Feat: Add Google OneNote Document Reader Support

community/document-readers/onenote-document-reader/pom.xml

@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2024-2025 the original author or authors.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.alibaba.cloud.ai</groupId>
+        <artifactId>spring-ai-alibaba</artifactId>
+        <version>${revision}</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>onenote-document-reader</artifactId>
+    <name>onenote-document-reader</name>
+    <description>onenote reader for Spring AI Alibaba</description>
+    <packaging>jar</packaging>
+    <url>https://github.com/alibaba/spring-ai-alibaba</url>
+    <scm>
+        <url>https://github.com/alibaba/spring-ai-alibaba</url>
+        <connection>git://github.com/alibaba/spring-ai-alibaba.git</connection>
+        <developerConnection>[email protected]:alibaba/spring-ai-alibaba.git</developerConnection>
+    </scm>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven-deploy-plugin.version>3.1.1</maven-deploy-plugin.version>
+        <jsoup-version>1.18.1</jsoup-version>
+    </properties>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>com.alibaba.cloud.ai</groupId>
+            <artifactId>spring-ai-alibaba-core</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.alibaba.cloud.ai</groupId>
+            <artifactId>document-parser-markdown</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>${jsoup-version}</version>
+        </dependency>
+
+        <!-- test dependencies -->
+        <dependency>
+            <groupId>org.springframework.ai</groupId>
+            <artifactId>spring-ai-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>io.micrometer</groupId>
+            <artifactId>micrometer-observation-test</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-deploy-plugin</artifactId>
+                <version>${maven-deploy-plugin.version}</version>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <repositories>
+        <repository>
+            <id>spring-milestones</id>
+            <name>Spring Milestones</name>
+            <url>https://repo.spring.io/milestone</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+</project>

community/document-readers/onenote-document-reader/src/main/java/com/alibaba/cloud/api/reader/onenote/OneNoteDocumentReader.java

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2024-2025 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alibaba.cloud.api.reader.onenote;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import org.jsoup.Jsoup;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.ai.document.Document;
+import org.springframework.ai.document.DocumentReader;
+import org.springframework.util.Assert;
+import org.springframework.util.StringUtils;
+
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Instant;
+import java.util.*;
+
+/**
+ * @author sparkle6979l
+ */
+public class OneNoteDocumentReader implements DocumentReader {
+
+    public static final String MICROSOFT_GRAPH_BASE_URL = "https://graph.microsoft.com/v1.0";
+
+    public static final String NOTEBOOK_ID_FILTER_PREFIX = "/me/onenote/pages/?$expand=parentNotebook&$filter=parentNotebook/id";
+
+    public static final String SECTION_ID_FILTER_PREFIX = "/me/onenote/pages/?$expand=parentSection&$filter=parentSection/id";
+
+    private static final Logger log = LoggerFactory.getLogger(OneNoteDocumentReader.class);
+
+    private final OneNoteResource oneNoteResource;
+
+    private final HttpClient client;
+
+    private final String accessToken;
+
+    public OneNoteDocumentReader(String accessToken, OneNoteResource oneNoteResource) {
+        this.accessToken = accessToken;
+        this.oneNoteResource = oneNoteResource;
+        this.client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_2).build();
+    }
+
+    /**
+     * Retrieves the content of a OneNote notebook by querying the Microsoft Graph API.
+     */
+    private List<String> getNoteBookContent(String accessToken, String notebookId) {
+        // Build the URI for fetching pages from the notebook
+        String uri = MICROSOFT_GRAPH_BASE_URL + NOTEBOOK_ID_FILTER_PREFIX
+                + "+eq+" + "'" + notebookId + "'";
+
+        // Get the page IDs from the notebook by querying the API
+        List<String> pageIdsFromNotebook = getOneNotePageIdsByURI(uri);
+
+        // Fetch the content for each page by its ID
+        return pageIdsFromNotebook.stream()
+                .map(id -> getPageContent(accessToken, id))
+                .toList();
+    }
+
+    /**
+     * Retrieves the content of a OneNote section by querying the Microsoft Graph API.
+     */
+    private List<String> getSectionContent(String accessToken, String sectionId) {
+        // Build the URI for fetching pages from the section
+        String uri = MICROSOFT_GRAPH_BASE_URL + SECTION_ID_FILTER_PREFIX
+                + "+eq+" + "'" + sectionId + "'";
+
+        // Get the page IDs from the notebook by querying the API
+        List<String> pageIdsBySection = getOneNotePageIdsByURI(uri);
+
+        // Fetch the content for each page by its ID
+        return pageIdsBySection.stream()
+                .map(id -> getPageContent(accessToken, id))
+                .toList();
+    }
+
+    private List<String> getOneNotePageIdsByURI(String uri) {
+        HttpRequest request = HttpRequest.newBuilder()
+                .header("Authorization", accessToken)
+                .header("Content-Type", "application/json")
+                .uri(URI.create(uri))
+                .GET()
+                .build();
+
+        try {
+            HttpResponse<String> response = this.client.send(request, HttpResponse.BodyHandlers.ofString());
+            Assert.isTrue(response.statusCode() == 200, "Failed to fetch pages information");
+            // Parse JSON response and extract page IDs
+            return parsePageIdsFromJson(response.body());
+        } catch (Exception e) {
+            throw new RuntimeException("Failed to get pages id", e);
+        }
+    }
+
+    /**
+     * Parses the JSON response and extracts page IDs
+     */
+    private List<String> parsePageIdsFromJson(String jsonResponse) {
+        JsonObject rootObject = JsonParser.parseString(jsonResponse).getAsJsonObject();
+        JsonArray valueArray = rootObject.getAsJsonArray("value");
+
+        return valueArray.asList().stream()
+                .map(jsonElement -> jsonElement.getAsJsonObject().get("id").getAsString())
+                .toList();
+    }
+
+    /**
+     * Retrieves the content of a specific OneNote page by querying the Microsoft Graph API.
+     */
+    private String getPageContent(String accessToken, String pageId) {
+        URI uri = URI.create(MICROSOFT_GRAPH_BASE_URL + "/me/onenote/pages/" + pageId + "/content");
+        HttpRequest request = HttpRequest.newBuilder()
+                .header("Authorization", accessToken)
+                .uri(uri)
+                .GET()
+                .build();
+        try {
+            HttpResponse<String> response = this.client.send(request, HttpResponse.BodyHandlers.ofString());
+            Assert.isTrue(response.statusCode() == 200, "Failed to fetch page blocks");
+            return parseHtmlContent(response.body());
+        } catch (Exception e) {
+            log.warn("Failed to get page content with token: {}, pageId: {}, {}", accessToken, pageId, e.getMessage(), e);
+            throw new RuntimeException("Failed to get page content", e);
+        }
+    }
+
+    @Override
+    public List<Document> get() {
+        // Get the access token
+        String accessToken = this.accessToken;
+        // Get the resource type and resource ID for the OneNote resource
+        OneNoteResource.ResourceType resourceType = this.oneNoteResource.getResourceType();
+        String resourceId = this.oneNoteResource.getResourceId();
+
+        // Fetch content based on the resource type (Notebook, Section, or Page)
+        List<String> content = switch (resourceType) {
+            case NOTEBOOK -> getNoteBookContent(accessToken, resourceId);
+            case SECTION -> getSectionContent(accessToken, resourceId);
+            case PAGE -> Collections.singletonList(getPageContent(accessToken, resourceId));
+        };
+
+        // Build metadata for the resource
+        Map<String, Object> metaData = buildMetadata(accessToken);
+
+        // Construct a list of Document objects
+        return content.stream()
+                .map(c -> new Document(c, metaData))
+                .toList();
+    }
+
+    private String parseHtmlContent(String htmlContent) {
+        // Parse the HTML content
+        org.jsoup.nodes.Document parseDoc = Jsoup.parse(htmlContent);
+
+        // Get title and text content, ensuring title is not empty
+        String title = parseDoc.title();
+        String text = parseDoc.text();
+
+        // Return title and content in a readable format
+        return title + (StringUtils.hasText(title) ? "" : "\n") + text;
+    }
+
+    /**
+     * Builds metadata for a given OneNote resource (Notebook, Section, or Page) by querying the Microsoft Graph API.
+     */
+    private Map<String, Object> buildMetadata(String accessToken) {
+        Map<String, Object> metadata = new HashMap<>();
+
+        String resourceId = this.oneNoteResource.getResourceId();
+        OneNoteResource.ResourceType resourceType = this.oneNoteResource.getResourceType();
+        String endpoint = switch (resourceType) {
+            case NOTEBOOK -> "/notebooks/";
+            case SECTION -> "/sections/";
+            case PAGE -> "/pages/";
+        };
+        String uriPath = MICROSOFT_GRAPH_BASE_URL + "/me/onenote" + endpoint + resourceId;
+        URI uri = URI.create(uriPath);
+
+        // Add basic metadata to the map (resource URI, type, and ID)
+        metadata.put(OneNoteResource.SOURCE, uriPath);
+        metadata.put("resourceType", resourceType.name());
+        metadata.put("resourceId", resourceId);
+
+        try {
+            HttpRequest request = HttpRequest.newBuilder()
+                    .header("Authorization", accessToken)
+                    .header("Content-Type", "application/json")
+                    .uri(uri)
+                    .GET()
+                    .build();
+
+            HttpResponse<String> response = this.client.send(request, HttpResponse.BodyHandlers.ofString());
+            Assert.isTrue(response.statusCode() == 200, "Failed to fetch page blocks");
+
+            // Parse the JSON response to extract relevant metadata fields
+            JsonObject jsonMetaData = JsonParser.parseString(response.body()).getAsJsonObject();
+
+            // Extract creation date and add to metadata if available
+            String createDateTime = Optional.ofNullable(jsonMetaData.get("createdDateTime"))
+                    .map(JsonElement::getAsString)
+                    .orElse(null);
+            if (StringUtils.hasText(createDateTime)) {
+                metadata.put("createdTime", Instant.parse(createDateTime).toEpochMilli());
+            }
+
+            // Extract last modified date and add to metadata if available
+            String lastModifiedDateTime = Optional.ofNullable(jsonMetaData.get("lastModifiedDateTime"))
+                    .map(JsonElement::getAsString)
+                    .orElse(null);
+            if (StringUtils.hasText(lastModifiedDateTime)) {
+                metadata.put("lastModifiedTime", Instant.parse(lastModifiedDateTime).toEpochMilli());
+            }
+
+            // Extract content URL and add to metadata if available
+            String contentURL = Optional.ofNullable(jsonMetaData.get("contentUrl"))
+                    .map(JsonElement::getAsString)
+                    .orElse(null);
+            if (StringUtils.hasText(contentURL)) {
+                metadata.put("contentURL", contentURL);
+            }
+
+        } catch (Exception e) {
+            log.warn("Failed to get page content with token: {}, resourceId: {}, resourceType: {}, {}",
+                    accessToken, resourceId, resourceType, e.getMessage(), e);
+            throw new RuntimeException("Failed to get page content", e);
+        }
+        return metadata;
+    }
+}