Skip to content

Commit

Permalink
Merge pull request #93 from redouane59/feat_full_search_academic
Browse files Browse the repository at this point in the history 
feat: full archive search
  • Loading branch information
@redouane59
redouane59 authored Dec 17, 2020
2 parents bd84ee5 + 0cc769b commit 422ce78
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 54 deletions.
14 changes: 13 additions & 1 deletion src/main/java/com/github/redouane59/twitter/ITwitterClientV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ public interface ITwitterClientV2 {
*/
TweetSearchResponse searchForTweetsWithin7days(String query, int maxResult, String nextToken);


/**
* Search tweets from last 7 days calling https://api.twitter.com/2/tweets/search
*
Expand All @@ -109,6 +108,19 @@ public interface ITwitterClientV2 {
*/
TweetSearchResponse searchForTweetsWithin7days(String query, LocalDateTime fromDate, LocalDateTime toDate, int maxResult, String nextToken);

/**
* Search archived tweets calling https://api.twitter.com/2/tweets/search/all
*
* @param query the search query
* @param fromDate the start date
* @param toDate the end date
* @param maxResult maximum 100 (default 10)
* @param nextToken the next_token given by the API to start from an index
* @return a TweetSearchResponse object containing a list of tweets and the next token
*/
TweetSearchResponse searchForTweetsFullArchive(String query, LocalDateTime fromDate, LocalDateTime toDate, int maxResult, String nextToken);


/**
* Stream using previous set up filters calling https://api.twitter.com/2/tweets/search/stream
*/
Expand Down
25 changes: 22 additions & 3 deletions src/main/java/com/github/redouane59/twitter/TwitterClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,12 @@ public TweetSearchResponse searchForTweetsWithin7days(String query, int maxResul
return this.searchForTweetsWithin7days(query, null, null, maxResult, nextToken);
}

@Override
public TweetSearchResponse searchForTweetsWithin7days(String query, LocalDateTime fromDate, LocalDateTime toDate, int maxResult, String nextToken) {
private TweetSearchResponse searchForTweets(String query,
LocalDateTime fromDate,
LocalDateTime toDate,
int maxResult,
String nextToken,
String searchUrl) {
Map<String, String> parameters = new HashMap<>();
parameters.put(QUERY, query);
parameters.put(MAX_RESULTS, String.valueOf(maxResult));
Expand All @@ -409,14 +413,28 @@ public TweetSearchResponse searchForTweetsWithin7days(String query, LocalDateTim
}
parameters.put("tweet.fields", ALL_TWEET_FIELDS);
Optional<TweetSearchResponseV2> tweetSearchV2DTO = this.requestHelperV2.getRequestWithParameters(
URLHelper.SEARCH_TWEET_7_DAYS_URL, parameters, TweetSearchResponseV2.class);
searchUrl, parameters, TweetSearchResponseV2.class);
if (tweetSearchV2DTO.isEmpty() || tweetSearchV2DTO.get().getData() == null) {
return new TweetSearchResponse(new ArrayList<>(), null);
}
List<Tweet> result = new ArrayList<>(tweetSearchV2DTO.get().getData());
return new TweetSearchResponse(result, tweetSearchV2DTO.get().getMeta().getNextToken());
}

@Override
public TweetSearchResponse searchForTweetsWithin7days(String query, LocalDateTime fromDate, LocalDateTime toDate, int maxResult, String nextToken) {
return this.searchForTweets(query, fromDate, toDate, maxResult, nextToken, URLHelper.SEARCH_TWEET_7_DAYS_URL);
}

@Override
public TweetSearchResponse searchForTweetsFullArchive(final String query,
final LocalDateTime fromDate,
final LocalDateTime toDate,
final int maxResult,
final String nextToken) {
return this.searchForTweets(query, fromDate, toDate, maxResult, nextToken, URLHelper.SEARCH_TWEET_FULL_ARCHIVE_URL);
}

@Override
public List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName) {
int count = 100;
Expand All @@ -442,6 +460,7 @@ public List<Tweet> searchForTweetsWithin30days(String query, LocalDateTime fromD
}

@Override
@Deprecated
public List<Tweet> searchForTweetsArchive(String query, LocalDateTime fromDate, LocalDateTime toDate, String envName) {
int count = 100;
Map<String, String> parameters = new HashMap<>();
Expand Down
97 changes: 49 additions & 48 deletions src/main/java/com/github/redouane59/twitter/helpers/URLHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,58 +10,59 @@
@Slf4j
public class URLHelper {

private static final String ROOT_URL_V1 = "https://api.twitter.com/1.1";
private static final String ROOT_URL_V2 = "https://api.twitter.com/2";
private static final String ROOT_URL_LABS_V2 = "https://api.twitter.com/labs/2";
private static final String IDS_JSON = "/ids.json?";
private static final String SCREEN_NAME = "screen_name";
private static final String ID = "id";
private static final String COUNT = "count";
private static final String LIST_JSON = "/list.json?";
private static final String SHOW_JSON = "/show.json?";
private static final String CREATE_JSON = "/create.json?";
private static final String DESTROY_JSON = "/destroy.json?";
private static final String RETWEETERS = "/retweeters";
private static final String FOLLOWERS = "/followers";
private static final String FRIENDS = "/friends";
private static final String STATUSES = "/statuses";
private static final String FRIENDSHIPS = "/friendships";
private static final String FAVORITES = "/favorites";
private static final String USERS = "/users";
private static final String TWEETS = "/tweets";
private static final String SEARCH = "/search";
private static final String SAMPLE = "/sample";
private static final String STREAM = "/stream";
private static final String THIRTY_DAYS = "/30day";
private static final String FULL_ARCHIVE = "/fullarchive";
private static final String ACCOUNT_ACTIVITY = "/account_activity/all";
private static final String WEBHOOKS = "/webhooks";
private static final String USER_ID = "user_id";
private static final String LOOKUP_JSON = "/lookup.json?";
private static final String USER_TIMELINE = "/user_timeline.json?";
private static final String JSON = ".json";
private static final String TRIM_USER = "trim_user=true";
private static final String EXCLUDE_RTS = "include_rts=false";
private static final String USER_FORMAT_DETAILED = "user.format=detailed";
private static final String TWEET_FORMAT_DETAILED = "tweet.format=detailed";
private static final String EXPANSIONS_RECENT_TWEET = "expansions=most_recent_tweet_id";
private static final String MAX_ID = "max_id";
private static final int MAX_COUNT = 200;
private static final int RETWEET_MAX_COUNT = 100;
private static final int MAX_LOOKUP = 100;
private static final String ROOT_URL_V1 = "https://api.twitter.com/1.1";
private static final String ROOT_URL_V2 = "https://api.twitter.com/2";
private static final String ROOT_URL_LABS_V2 = "https://api.twitter.com/labs/2";
private static final String IDS_JSON = "/ids.json?";
private static final String SCREEN_NAME = "screen_name";
private static final String ID = "id";
private static final String COUNT = "count";
private static final String LIST_JSON = "/list.json?";
private static final String SHOW_JSON = "/show.json?";
private static final String CREATE_JSON = "/create.json?";
private static final String DESTROY_JSON = "/destroy.json?";
private static final String RETWEETERS = "/retweeters";
private static final String FOLLOWERS = "/followers";
private static final String FRIENDS = "/friends";
private static final String STATUSES = "/statuses";
private static final String FRIENDSHIPS = "/friendships";
private static final String FAVORITES = "/favorites";
private static final String USERS = "/users";
private static final String TWEETS = "/tweets";
private static final String SEARCH = "/search";
private static final String SAMPLE = "/sample";
private static final String STREAM = "/stream";
private static final String THIRTY_DAYS = "/30day";
private static final String FULL_ARCHIVE = "/fullarchive";
private static final String ACCOUNT_ACTIVITY = "/account_activity/all";
private static final String WEBHOOKS = "/webhooks";
private static final String USER_ID = "user_id";
private static final String LOOKUP_JSON = "/lookup.json?";
private static final String USER_TIMELINE = "/user_timeline.json?";
private static final String JSON = ".json";
private static final String TRIM_USER = "trim_user=true";
private static final String EXCLUDE_RTS = "include_rts=false";
private static final String USER_FORMAT_DETAILED = "user.format=detailed";
private static final String TWEET_FORMAT_DETAILED = "tweet.format=detailed";
private static final String EXPANSIONS_RECENT_TWEET = "expansions=most_recent_tweet_id";
private static final String MAX_ID = "max_id";
private static final int MAX_COUNT = 200;
private static final int RETWEET_MAX_COUNT = 100;
private static final int MAX_LOOKUP = 100;
private static final String
ALL_USER_FIELDS =
ALL_USER_FIELDS =
"user.fields=id,created_at,username,name,location,url,verified,profile_image_url,public_metrics,pinned_tweet_id,description,protected";
private static final String
ALL_TWEET_FIELDS =
ALL_TWEET_FIELDS =
"tweet.fields=attachments,author_id,created_at,entities,geo,id,in_reply_to_user_id,lang,possibly_sensitive,public_metrics,referenced_tweets,source,text,withheld,context_annotations,conversation_id";
public static final String LAST_TWEET_LIST_URL = ROOT_URL_V1 + STATUSES + USER_TIMELINE;
public static final String RATE_LIMIT_URL = ROOT_URL_V1 + "/application/rate_limit_status.json";
public static final String SEARCH_TWEET_STANDARD_URL = ROOT_URL_V1 + SEARCH + TWEETS + JSON;
public static final String SEARCH_TWEET_7_DAYS_URL = ROOT_URL_V2 + TWEETS + SEARCH + "/recent";
public static final String GET_BEARER_TOKEN_URL = "https://api.twitter.com/oauth2/token";
public static final String GET_OAUTH1_TOKEN_URL = "https://api.twitter.com/oauth/request_token";
public static final String GET_OAUTH1_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token";
public static final String LAST_TWEET_LIST_URL = ROOT_URL_V1 + STATUSES + USER_TIMELINE;
public static final String RATE_LIMIT_URL = ROOT_URL_V1 + "/application/rate_limit_status.json";
public static final String SEARCH_TWEET_STANDARD_URL = ROOT_URL_V1 + SEARCH + TWEETS + JSON;
public static final String SEARCH_TWEET_7_DAYS_URL = ROOT_URL_V2 + TWEETS + SEARCH + "/recent";
public static final String SEARCH_TWEET_FULL_ARCHIVE_URL = ROOT_URL_V2 + TWEETS + SEARCH + "/all";
public static final String GET_BEARER_TOKEN_URL = "https://api.twitter.com/oauth2/token";
public static final String GET_OAUTH1_TOKEN_URL = "https://api.twitter.com/oauth/request_token";
public static final String GET_OAUTH1_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token";


public String getSearchTweet30DaysUrl(String envName) {
Expand Down
12 changes: 11 additions & 1 deletion src/test/java/com/github/redouane59/twitter/nrt/ITwitterClientV2Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.github.redouane59.twitter.dto.tweet.TweetSearchResponse;
import com.github.redouane59.twitter.dto.tweet.TweetType;
import com.github.redouane59.twitter.dto.user.User;
import com.github.redouane59.twitter.helpers.ConverterHelper;
import com.github.redouane59.twitter.signature.TwitterCredentials;
import java.io.File;
import java.io.IOException;
Expand All @@ -29,7 +30,7 @@ public class ITwitterClientV2Test {

@BeforeAll
public static void init() throws IOException {
String credentialPath = "C:/Users/Perso/Documents/GitHub/twitter-credentials.json";
String credentialPath = "C:/Users/Perso/Documents/GitHub/twitter-credentials - RBA.json";
twitterClient = new TwitterClient(TwitterClient.OBJECT_MAPPER
.readValue(new File(credentialPath), TwitterCredentials.class));
}
Expand Down Expand Up @@ -149,6 +150,15 @@ public void testSearchTweets7daysWithNexTokenAndCount() {
assertNotNull(result2.getNextToken());
}

@Test
public void testSearchTweetsFullArchiveWithNexTokenAndCount() {
TweetSearchResponse
result =
twitterClient.searchForTweetsFullArchive("@TwitterSupport", ConverterHelper.dayBeforeNow(150), ConverterHelper.dayBeforeNow(1), 100, null);
assertTrue(result.getTweets().size() > 10);
assertNotNull(result.getNextToken());
}

@Test
public void testGetTweetType() {
assertEquals(TweetType.QUOTED, twitterClient.getTweet("1267115291991068673").getTweetType());
Expand Down
8 changes: 7 additions & 1 deletion src/test/java/com/github/redouane59/twitter/unit/UrlHelperTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,12 @@ public void testSearch7DaysUrl() {
URLHelper.SEARCH_TWEET_7_DAYS_URL);
}

@Test
public void testSearchFullArchiveUrl() {
assertEquals("https://api.twitter.com/2/tweets/search/all",
URLHelper.SEARCH_TWEET_FULL_ARCHIVE_URL);
}

@Test
public void testGetBearerTokenUrl() {
assertEquals("https://api.twitter.com/oauth2/token",
Expand Down Expand Up @@ -207,7 +213,7 @@ public void testSampledStreamUrl() {
}

@Test
public void testSearchFullArchiveUrl() {
public void testSearchFullArchiveUrlv1() {
assertEquals("https://api.twitter.com/1.1/tweets/search/fullarchive/dev.json", urlHelper.getSearchTweetFullArchiveUrl("dev"));
}

Expand Down

0 comments on commit 422ce78

Please sign in to comment.