From 05f363ec64a66a8a9e438646ffdb0125a8b89a52 Mon Sep 17 00:00:00 2001 From: Stephen Connolly Date: Tue, 18 Jul 2023 08:21:32 +0100 Subject: [PATCH] feat: add support for chunked uploading (#448) tested manually with GIF images, but no automated tests in this I hereby declare that this contribution is licensed under Apache License Version 2 --- pom.xml | 1 + .../redouane59/twitter/ITwitterClientV1.java | 14 ++- .../redouane59/twitter/TwitterClient.java | 85 ++++++++++++++++--- .../dto/tweet/UploadMediaProcessingError.java | 17 ++++ .../dto/tweet/UploadMediaProcessingInfo.java | 23 +++++ .../dto/tweet/UploadMediaResponse.java | 2 + .../helpers/AbstractRequestHelper.java | 6 +- .../twitter/helpers/RequestHelper.java | 12 +++ .../redouane59/twitter/helpers/URLHelper.java | 13 ++- 9 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java create mode 100644 src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java diff --git a/pom.xml b/pom.xml index 5241126c..6f634153 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ false + -Xdoclint:none jar diff --git a/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java b/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java index 67c6b5dc..9f10b989 100644 --- a/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java +++ b/src/main/java/io/github/redouane59/twitter/ITwitterClientV1.java @@ -12,8 +12,10 @@ import io.github.redouane59.twitter.dto.tweet.Tweet; import io.github.redouane59.twitter.dto.tweet.UploadMediaResponse; import java.io.File; +import java.io.InputStream; import java.time.LocalDateTime; import java.util.List; +import java.util.Optional; public interface ITwitterClientV1 { @@ -113,6 +115,16 @@ public interface ITwitterClientV1 { */ UploadMediaResponse uploadMedia(File media, MediaCategory mediaCategory); + /** + * Upload a media calling https://upload.twitter.com/1.1/media/upload in chunked mode. + */ + Optional uploadChunkedMedia(String mediaName, long size, InputStream data, MediaCategory mediaCategory); + + /** + * Upload a media calling https://upload.twitter.com/1.1/media/upload in chunked mode. + */ + Optional uploadChunkedMedia(File media, MediaCategory mediaCategory); + /** * Creates a collection of tweets. See https://api.twitter.com/1.1/collections/create.json * @@ -190,4 +202,4 @@ public interface ITwitterClientV1 { @Deprecated DmEvent postDm(String text, String userId); -} \ No newline at end of file +} diff --git a/src/main/java/io/github/redouane59/twitter/TwitterClient.java b/src/main/java/io/github/redouane59/twitter/TwitterClient.java index 192abb23..a592bf3d 100644 --- a/src/main/java/io/github/redouane59/twitter/TwitterClient.java +++ b/src/main/java/io/github/redouane59/twitter/TwitterClient.java @@ -39,21 +39,9 @@ import io.github.redouane59.twitter.dto.stream.StreamRules; import io.github.redouane59.twitter.dto.stream.StreamRules.StreamMeta; import io.github.redouane59.twitter.dto.stream.StreamRules.StreamRule; -import io.github.redouane59.twitter.dto.tweet.HiddenResponse; +import io.github.redouane59.twitter.dto.tweet.*; import io.github.redouane59.twitter.dto.tweet.HiddenResponse.HiddenData; -import io.github.redouane59.twitter.dto.tweet.LikeResponse; -import io.github.redouane59.twitter.dto.tweet.MediaCategory; -import io.github.redouane59.twitter.dto.tweet.RetweetResponse; -import io.github.redouane59.twitter.dto.tweet.Tweet; -import io.github.redouane59.twitter.dto.tweet.TweetCountsList; -import io.github.redouane59.twitter.dto.tweet.TweetList; import io.github.redouane59.twitter.dto.tweet.TweetList.TweetMeta; -import io.github.redouane59.twitter.dto.tweet.TweetParameters; -import io.github.redouane59.twitter.dto.tweet.TweetSearchResponseV1; -import io.github.redouane59.twitter.dto.tweet.TweetV1; -import io.github.redouane59.twitter.dto.tweet.TweetV1Deserializer; -import io.github.redouane59.twitter.dto.tweet.TweetV2; -import io.github.redouane59.twitter.dto.tweet.UploadMediaResponse; import io.github.redouane59.twitter.dto.user.FollowBody; import io.github.redouane59.twitter.dto.user.User; import io.github.redouane59.twitter.dto.user.UserActionResponse; @@ -70,6 +58,8 @@ import io.github.redouane59.twitter.signature.TwitterCredentials; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -1399,6 +1389,75 @@ public UploadMediaResponse uploadMedia(File imageFile, MediaCategory mediaCatego return requestHelperV1.uploadMedia(url, imageFile, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new); } + @Override + public Optional uploadChunkedMedia(String mediaName, long size, InputStream data, MediaCategory mediaCategory) { + try { + String type = URLConnection.guessContentTypeFromName(mediaName); + String url = urlHelper.getChunkedUploadMediaUrl(); + Map parameters = new HashMap<>(); + parameters.put("command", "INIT"); + parameters.put("total_bytes", Long.toString(size)); + parameters.put("media_type", type); + parameters.put("medai_category", mediaCategory.label); + UploadMediaResponse initRsp = requestHelperV1.postRequest(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new); + + parameters.clear(); + parameters.put("command", "APPEND"); + parameters.put("media_id", initRsp.getMediaId()); + + byte[] buf = new byte[(int) Math.min(size, 5 * 1024 * 1024L)]; // 5MB max chunk size + int segmentIndex = 0; + int count; + try { + while ((count = data.read(buf)) > 0) { + parameters.put("segment_index", Integer.toString(segmentIndex++)); + requestHelperV1.uploadChunkedMedia(url, parameters, buf, 0, count, Void.class); + } + } catch (IOException ex) { + LOGGER.error("Error occupied on reading media", ex); + return Optional.empty(); + } + + parameters.clear(); + parameters.put("command", "FINALIZE"); + parameters.put("media_id", initRsp.getMediaId()); + + UploadMediaResponse rsp = requestHelperV1.postRequest(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new); + UploadMediaProcessingInfo processing; + while ((processing = rsp.getProcessingInfo()) != null && processing.getState().equals("pending")) { + try { + Thread.sleep(processing.getCheckAfterSecs() * 1000L); + } catch (InterruptedException ex) { + LOGGER.error("Error occupied on waiting media processing", ex); + } + + parameters.clear(); + parameters.put("command", "STATUS"); + parameters.put("media_id", initRsp.getMediaId()); + + rsp = requestHelperV1.getRequestWithParameters(url, parameters, UploadMediaResponse.class).orElseThrow(NoSuchElementException::new); + } + + return Optional.of(rsp); + } finally { + try { + data.close(); + } catch (IOException ex) { + LOGGER.error("Error occupied on closing media stream", ex); + } + } + } + + @Override + public Optional uploadChunkedMedia(File imageFile, MediaCategory mediaCategory) { + try { + return uploadChunkedMedia(imageFile.getName(), imageFile.length(), Files.newInputStream(imageFile.toPath()), mediaCategory); + } catch (IOException ex) { + LOGGER.error("Error occupied on reading media", ex); + return Optional.empty(); + } + } + @Override public CollectionsResponse collectionsCreate(String name, String description, String collectionUrl, TimeLineOrder timeLineOrder) { String url = getUrlHelper().getCollectionsCreateUrl(); diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java new file mode 100644 index 00000000..c50969fb --- /dev/null +++ b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingError.java @@ -0,0 +1,17 @@ +package io.github.redouane59.twitter.dto.tweet; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +public class UploadMediaProcessingError { + + private int code; + private String name; + private String message; + +} diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java new file mode 100644 index 00000000..02770aba --- /dev/null +++ b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaProcessingInfo.java @@ -0,0 +1,23 @@ +package io.github.redouane59.twitter.dto.tweet; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +public class UploadMediaProcessingInfo { + + @JsonProperty("state") + private String state; + @JsonProperty("check_after_secs") + private int checkAfterSecs; + @JsonProperty("progress_percent") + private int progressPercent; + @JsonProperty("media_key") + private String mediaKey; + private UploadMediaProcessingError error; + +} diff --git a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java index 04255cd2..7866ccdf 100644 --- a/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java +++ b/src/main/java/io/github/redouane59/twitter/dto/tweet/UploadMediaResponse.java @@ -17,5 +17,7 @@ public class UploadMediaResponse { private int size; @JsonProperty("media_key") private String mediaKey; + @JsonProperty("processing_info") + private UploadMediaProcessingInfo processingInfo; } diff --git a/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java index 84f5862e..d31fedd5 100644 --- a/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java +++ b/src/main/java/io/github/redouane59/twitter/helpers/AbstractRequestHelper.java @@ -131,7 +131,9 @@ public Optional makeRequest(OAuthRequest request, boolean signRequired, C } else if (response.getCode() < 200 || response.getCode() > 299) { logApiError(request.getVerb().name(), request.getUrl(), stringResponse, response.getCode()); } - result = convert(stringResponse, classType); + if (!Void.class.equals(classType)) { + result = convert(stringResponse, classType); + } } catch (IOException ex) { LOGGER.error("Error occupied on executing request", ex); } @@ -150,4 +152,4 @@ protected T convert(String json, Class targetClass) throws Json public abstract Optional getRequestWithParameters(String url, Map parameters, Class classType); -} \ No newline at end of file +} diff --git a/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java index 3b437396..839b8773 100644 --- a/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java +++ b/src/main/java/io/github/redouane59/twitter/helpers/RequestHelper.java @@ -51,6 +51,18 @@ public Optional uploadMedia(String url, String fileName, byte[] data, Cla return makeRequest(request, true, classType); } + public Optional uploadChunkedMedia(String url, Map parameters, byte[] media, int off, int len, Class classType) { + OAuthRequest request = new OAuthRequest(Verb.POST, url); + if (parameters != null) { + for (Map.Entry param : parameters.entrySet()) { + request.addQuerystringParameter(param.getKey(), param.getValue()); + } + } + request.initMultipartPayload(); + request.addBodyPartPayloadInMultipartPayload(new FileByteArrayBodyPartPayload("application/octet-stream", media, off, len, "media")); + return makeRequest(request, true, classType); + } + public Optional putRequest(String url, String body, Class classType) { return makeRequest(Verb.PUT, url, null, body, true, classType); } diff --git a/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java b/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java index 22824876..816530e6 100644 --- a/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java +++ b/src/main/java/io/github/redouane59/twitter/helpers/URLHelper.java @@ -3,6 +3,10 @@ import io.github.redouane59.twitter.dto.tweet.MediaCategory; import lombok.Getter; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + public class URLHelper { public static final int MAX_LOOKUP = 100; @@ -11,6 +15,7 @@ public class URLHelper { public static final String GET_OAUTH1_TOKEN_URL = "https://api.twitter.com/oauth/request_token"; public static final String GET_OAUTH1_ACCESS_TOKEN_URL = "https://api.twitter.com/oauth/access_token"; private static final String ROOT_URL_V1 = "https://api.twitter.com/1.1"; + private static final String UPLOAD_URL_V1 = "https://upload.twitter.com/1.1/media/upload.json"; public static final String RATE_LIMIT_URL = ROOT_URL_V1 + "/application/rate_limit_status.json"; // v1 legacy private static final String IDS_JSON = "/ids.json?"; @@ -197,7 +202,13 @@ public String getUserMentionsUrl(String userId) { } public String getUploadMediaUrl(MediaCategory mediaCategory) { - return "https://upload.twitter.com/1.1/media/upload.json?media_category=" + mediaCategory.label; + return UPLOAD_URL_V1 + + "?media_category=" + + mediaCategory.label; + } + + public String getChunkedUploadMediaUrl() { + return UPLOAD_URL_V1; } public String getCollectionsCreateUrl() {