From 11b9548b4f8ece509fcea3c5c2a487fd209e1580 Mon Sep 17 00:00:00 2001 From: bjwswang Date: Tue, 12 Sep 2023 16:40:21 +0800 Subject: [PATCH] feat: add a cli to load documents and chat with similarity search Signed-off-by: bjwswang --- Makefile | 5 ++ arctl/README.md | 172 ++++++++++++++++++++++++++++++++++++++ arctl/chat.go | 213 ++++++++++++++++++++++++++++++++++++++++++++++++ arctl/load.go | 137 +++++++++++++++++++++++++++++++ arctl/main.go | 42 ++++++++++ go.mod | 7 ++ go.sum | 26 ++++++ 7 files changed, 602 insertions(+) create mode 100644 arctl/README.md create mode 100644 arctl/chat.go create mode 100644 arctl/load.go create mode 100644 arctl/main.go diff --git a/Makefile b/Makefile index e0d32fc74..e04ef9fdb 100644 --- a/Makefile +++ b/Makefile @@ -233,3 +233,8 @@ catalog-build: opm ## Build a catalog image. .PHONY: catalog-push catalog-push: ## Push a catalog image. $(MAKE) docker-push IMG=$(CATALOG_IMG) + +# CLI Arcadia +.PHONY: arctl +arctl: fmt vet ## Build manager binary. + go build -o bin/arctl arctl/*.go \ No newline at end of file diff --git a/arctl/README.md b/arctl/README.md new file mode 100644 index 000000000..b5a8d778a --- /dev/null +++ b/arctl/README.md @@ -0,0 +1,172 @@ +# arctl + +arctl(arcadia command line tool) + +```shell +``` + +## Quick Install + +```shell +cd arcadia +make arctl +``` +If build succeeded, `arctl` will be built into `bin/arctl` under `arcadia` + +```shell +❯ ./bin/arctl -h +Command line tools for Arcadia + +Usage: + arctl [usage] [flags] + arctl [command] + +Available Commands: + load Load documents into VectorStore + chat Do LLM chat with similarity search(optional) + completion Generate the autocompletion script for the specified shell + help Help about any command + +Flags: + -h, --help help for arctl + +Use "arctl [command] --help" for more information about a command. +``` + +## Usage +### Load documents into vector store + +```shell +❯ ./bin/arctl load -h +Load documents into VectorStore + +Usage: + arctl load [usage] [flags] + +Flags: + --chunk-overlap int chunk overlap for embedding (default 30) + --chunk-size int chunk size for embedding (default 300) + --document string path of the document to load + --document-language string language of the document(Only text,html,csv supported now) (default "text") + --embedding-llm-apikey string apiKey to access embedding service + --embedding-llm-type string llm type to use(Only zhipuai,openai supported now) (default "zhipuai") + -h, --help help for load + --namespace string namespace/collection of the document to load into (default "arcadia") + --vector-store string vector stores to use(Only chroma supported now) (default "http://localhost:8000") +``` + +Required Arguments: +- `--embedding-llm-apikey` +- `--document` + +For example: +```shell +./bin/arctl load --embedding-llm-apikey 26b2bc55fae40752055cadfc4792f9de.wagA4NIwg5aZJWhm --document ./README.md +``` + + +### Chat with LLM +```shell +❯ ./bin/arctl chat -h +Do LLM chat with similarity search(optional) + +Usage: + arctl chat [usage] [flags] + +Flags: + --chat-llm-apikey string apiKey to access embedding service + --chat-llm-type string llm type to use(Only zhipuai,openai supported now) (default "zhipuai") + --embedding-llm-apikey string apiKey to access embedding service.Must required when embedding similarity search is enabled + --embedding-llm-type string llm type to use(Only zhipuai,openai supported now) (default "zhipuai") + --enable-embedding-search enable embedding similarity search + -h, --help help for chat + --method string Invoke method used when access LLM service(invoke/sse-invoke) (default "sse-invoke") + --model string which model to use: chatglm_lite/chatglm_std/chatglm_pro (default "chatglm_lite") + --namespace string namespace/collection to query from (default "arcadia") + --num-docs int number of documents to be returned with SimilarSearch (default 3) + --question string question text to be asked + --score-threshold float score threshold for similarity search(Higher is better) + --temperature float32 temperature for chat (default 0.95) + --top-p float32 top-p for chat (default 0.7) + --vector-store string vector stores to use(Only chroma supported now) (default "http://localhost:8000") +``` + +Now `arctl chat` has two modes which is controlled by flag `--enable-embedding-search`: +- normal chat without embedding search(default) +- enable similarity search with embedding + +#### Normal chat(Without embedding) + +```shell +./bin/arctl chat --chat-llm-apikey 26b2bc55fae40752055cadfc4792f9de.wagA4NIwg5aZJWhm --model chatglm_pro --question "介绍一下Arcadia" +``` + +Required Arguments: +- `--chat-llm-apikey` +- `--question` + + +**Output:** +```shell +Prompts: [{user 介绍一下Arcadia}] + Arcadia（阿卡迪亚）是一个知名的游戏开发团队，成立于 2007 年，总部位于美国加州洛杉矶。Arcadia 主要致力于创作高质量的社交游戏和虚拟世界，旨在为玩家提供一个沉浸式的游戏体验。他们的作品在全球范围内拥有大量的玩家，其中包括许多虚拟世界和模拟经营类的游戏。 + +Arcadia 的创始人兼 CEO 是 Raph Koster，他是一位在游戏行业拥有丰富经验的游戏设计师。在成立 Arcadia 之前，他曾担任过多家知名游戏公司的要职，如索尼在线娱乐、艺电等。 + +Arcadia 开发的游戏中最知名的作品之一是《Second Life》（第二人生），这是一款非常受欢迎的虚拟世界游戏，玩家可以在游戏中创建自己的虚拟角色、探索环境、与其他玩家互动等。这款游戏自推出以来获得了许多奖项，被誉为虚拟世界游戏的典范。 + +除了《Second Life》之外，Arcadia 还开发了许多其他游戏，如《Champions Online》、《Star Wars: Galaxies》等。此外，他们还为其他游戏公司提供游戏开发咨询和技术支持服务。 + +总的来说，Arcadia 是一家在游戏行业具有较高声誉和影响力的公司，他们不断推出新的游戏作品，为玩家带来更多精彩的游戏体验。 +``` + +#### Enable Similarity Search + +```shell +./bin/arctl chat --enable-embedding-search --chat-llm-apikey 26b2bc55fae40752055cadfc4792f9de.wagA4NIwg5aZJWhm --embedding-llm-apikey 26b2bc55fae40752055cadfc4792f9de.wagA4NIwg5aZJWhm --model chatglm_pro --num-docs 10 --question "介绍一下Arcadia" +``` + +Required Arguments: +- `--enable-embedding-search` +- `--embedding-llm-apikey` +- `--chat-llm-apikey` +- `--question` + +**Output:** +```shell +Prompts: [{user + 我将要询问一些问题，希望你仅使用我提供的上下文信息回答。 + 请不要在回答中添加其他信息。 + 若我提供的上下文不足以回答问题, + 请回复"我不确定"，再做出适当的猜测。 + 请将回答内容分割为适于阅读的段落。 + } {assistant + 好的，我将尝试仅使用你提供的上下文信息回答，并在信息不足时提供一些合理推测。 + } {user 我的问题是: 介绍一下Arcadia. 以下是我提供的上下文:--- sidebar_position: 1 --- # 介绍## Contribute to Arcadia If you want to contribute to Arcadia, refer to [contribute guide](CONTRIBUTING.md). ## Support If you need support, start with the troubleshooting guide, or create GitHub [issues](https://github.com/kubeagi/arcadia/issues/new)```of developers contributing to its continued development and improvement. It is available on a variety of platforms, including Linux, Windows, and 移动设备，and can be deployed on-premises, in the cloud, or in a hybrid# Arcadia Our vision is to make it easier for cloud-native applications to integrate with AI, thereby making the cloud more intelligent and impactful. ## Quick start 1. Install arcadia operatormaintained by the Cloud Native Computing Foundation (CNCF).\\n\\nKubernetes provides a platform-as-a-service (PaaS) model, which allows developers to deploy, run, and scale containerized applications with minimal configuration and effort. It does this by abstracting the underlying infrastructure andunderlying infrastructure and providing a common set of APIs and tools that can be used to deploy, manage, and scale applications consistently across different environments.\\n\\nKubernetes is widely adopted by organizations of all sizes and has a large, active community of developers contributingin the cloud, or in a hybrid environment.\"","role":"assistant"}],"request_id":"7865480399259975113","task_id":"7865480399259975113","task_status":"SUCCESS","usage":{"total_tokens":203}},"msg":"操作成功","success":true}[内核](https://github.com/kubebb/core)基于[kubernetes## Packages To enhace the AI capability in Golang,we developed some packages. ### LLMs - ✅ [ZhiPuAI(智谱AI)](https://github.com/kubeagi/arcadia/tree/main/pkg/llms/zhipuai) - [example](https://github.com/kubeagi/arcadia/blob/main/examples/zhipuai/main.go) ### Embeddings}] + Arcadia 是一个项目，旨在使云原生应用程序更容易与 AI 集成，从而使云更加智能和有影响力。该项目由 Cloud Native Computing Foundation（CNCF）维护，并可在多种平台上使用，包括 Linux、Windows 和移动设备。它可以通过本地部署、云部署或混合部署。 + +Kubernetes 是一个提供平台即服务（PaaS）模型的系统，它使开发人员能够以最小的配置和努力来部署、运行和扩展容器化应用程序。它通过抽象底层基础设施，并提供一组通用的 API 和工具来实现这一目标，这些 API 和工具可以用于在不同的环境中一致地部署、管理和扩展应用程序。 + +Arcadia 项目包含了一些用于增强 Golang 中 AI 功能的软件包。其中，智谱 AI（ZhiPuAI）是一个 LLM（大型语言模型）软件包，提供了示例代码供开发者参考。 +``` + +## Limitations + +1. Vector Store + +- ✅ chromadb + +2. Document Types + +- ✅ text +- ✅ html +- ✅ csv + +3. Embedding Service + +- ✅ zhipuai +- ✅ openai + +4. LLM Service + +- ✅ zhipuai diff --git a/arctl/chat.go b/arctl/chat.go new file mode 100644 index 000000000..f1fab52ba --- /dev/null +++ b/arctl/chat.go @@ -0,0 +1,213 @@ +/* +Copyright 2023 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "fmt" + + zhipuaiembeddings "github.com/kubeagi/arcadia/pkg/embeddings/zhipuai" + "github.com/kubeagi/arcadia/pkg/llms" + "github.com/kubeagi/arcadia/pkg/llms/zhipuai" + "github.com/kubeagi/arcadia/pkg/vectorstores/chromadb" + "github.com/spf13/cobra" + "github.com/tmc/langchaingo/embeddings" + openaiEmbeddings "github.com/tmc/langchaingo/embeddings/openai" + "github.com/tmc/langchaingo/schema" + "github.com/tmc/langchaingo/vectorstores" +) + +var ( + question string + // chat with LLM + chatLLMType string + chatAPIKey string + model string + method string + temperature float32 + topP float32 + + // similarity search + enableSimilaritySearch bool + scoreThreshold float64 + numDocs int + + // promptsWithSimilaritySearch = []zhipuai.Prompt{ + // { + // Role: zhipuai.User, + // Content: `Hi there, I am going to ask you a question, which I would like you to answer + // based only on the provided context, and not any other information. + // If there is not enough information in the context to answer the question,'say \"I am not sure\", then try to make a guess.' + // Break your answer up into nicely readable paragraphs.`, + // }, + // { + // Role: zhipuai.Assistant, + // Content: "Sure.Please provide your documents.", + // }, + // } + promptsWithSimilaritySearchCN = []zhipuai.Prompt{ + { + Role: zhipuai.User, + Content: ` + 我将要询问一些问题，希望你仅使用我提供的上下文信息回答。 + 请不要在回答中添加其他信息。 + 若我提供的上下文不足以回答问题, + 请回复"我不确定"，再做出适当的猜测。 + 请将回答内容分割为适于阅读的段落。 + `, + }, + { + Role: zhipuai.Assistant, + Content: ` + 好的，我将尝试仅使用你提供的上下文信息回答，并在信息不足时提供一些合理推测。 + `, + }, + } +) + +func NewChatCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "chat [usage]", + Short: "Do LLM chat with similarity search(optional)", + RunE: func(cmd *cobra.Command, args []string) error { + var docs []schema.Document + var err error + + if enableSimilaritySearch { + docs, err = SimilaritySearch(context.Background()) + if err != nil { + return err + } + fmt.Printf("similarDocs: %v \n", docs) + } + + return Chat(context.Background(), docs) + }, + } + + // For similarity search + cmd.Flags().BoolVar(&enableSimilaritySearch, "enable-embedding-search", false, "enable embedding similarity search") + cmd.Flags().StringVar(&embeddingLLMType, "embedding-llm-type", string(llms.ZhiPuAI), "llm type to use(Only zhipuai,openai supported now)") + cmd.Flags().StringVar(&embeddingLLMApiKey, "embedding-llm-apikey", "", "apiKey to access embedding service.Must required when embedding similarity search is enabled") + cmd.Flags().StringVar(&vectorStore, "vector-store", "http://localhost:8000", "vector stores to use(Only chroma supported now)") + cmd.Flags().StringVar(&nameSpace, "namespace", "arcadia", "namespace/collection to query from") + cmd.Flags().Float64Var(&scoreThreshold, "score-threshold", 0, "score threshold for similarity search(Higher is better)") + cmd.Flags().IntVar(&numDocs, "num-docs", 5, "number of documents to be returned with SimilarSearch") + + cmd.Flags().StringVar(&question, "question", "", "question text to be asked") + + // For LLM chat + cmd.Flags().StringVar(&chatLLMType, "chat-llm-type", string(llms.ZhiPuAI), "llm type to use(Only zhipuai,openai supported now)") + cmd.Flags().StringVar(&chatAPIKey, "chat-llm-apikey", "", "apiKey to access embedding service") + cmd.PersistentFlags().StringVar(&model, "model", string(zhipuai.ZhiPuAILite), "which model to use: chatglm_lite/chatglm_std/chatglm_pro") + cmd.PersistentFlags().StringVar(&method, "method", "sse-invoke", "Invoke method used when access LLM service(invoke/sse-invoke)") + cmd.PersistentFlags().Float32Var(&temperature, "temperature", 0.95, "temperature for chat") + cmd.PersistentFlags().Float32Var(&topP, "top-p", 0.7, "top-p for chat") + + cmd.MarkFlagRequired("chat-llm-apikey") + cmd.MarkFlagRequired("question") + + return cmd +} + +func SimilaritySearch(ctx context.Context) ([]schema.Document, error) { + var embedder embeddings.Embedder + var err error + + if embeddingLLMApiKey == "" { + return nil, errors.New("embedding-llm-apikey is required when embedding similarity search is enabled") + } + + switch embeddingLLMType { + case "zhipuai": + embedder, err = zhipuaiembeddings.NewZhiPuAI( + zhipuaiembeddings.WithClient(*zhipuai.NewZhiPuAI(embeddingLLMApiKey)), + ) + if err != nil { + return nil, err + } + case "openai": + embedder, err = openaiEmbeddings.NewOpenAI() + if err != nil { + return nil, err + } + default: + return nil, errors.New("unsupported embedding type") + } + + chroma, err := chromadb.New( + chromadb.WithURL(vectorStore), + chromadb.WithEmbedder(embedder), + chromadb.WithNameSpace(nameSpace), + ) + if err != nil { + return nil, err + } + + return chroma.SimilaritySearch(ctx, question, numDocs, vectorstores.WithScoreThreshold(scoreThreshold)) +} + +func Chat(ctx context.Context, similarDocs []schema.Document) error { + // Only for zhipuai + client := zhipuai.NewZhiPuAI(chatAPIKey) + + params := zhipuai.DefaultModelParams() + params.Model = zhipuai.Model(model) + params.Method = zhipuai.Method(method) + params.Temperature = temperature + params.TopP = topP + + var prompts []zhipuai.Prompt + if len(similarDocs) != 0 { + var docString string + for _, doc := range similarDocs { + docString += doc.PageContent + } + prompts = append(prompts, promptsWithSimilaritySearchCN...) + prompts = append(prompts, zhipuai.Prompt{ + Role: zhipuai.User, + Content: fmt.Sprintf("我的问题是: %s. 以下是我提供的上下文:%s", question, docString), + }) + } else { + prompts = append(prompts, zhipuai.Prompt{ + Role: zhipuai.User, + Content: question, + }) + } + + fmt.Printf("Prompts: %v \n", prompts) + params.Prompt = prompts + if params.Method == zhipuai.ZhiPuAIInvoke { + resp, err := client.Invoke(params) + if err != nil { + return err + } + if resp.Code != 200 { + return fmt.Errorf("chat failed: %s", resp.String()) + } + fmt.Println(resp.Data.Choices[0].Content) + return nil + } + + err := client.SSEInvoke(params, nil) + if err != nil { + return err + } + + return nil +} diff --git a/arctl/load.go b/arctl/load.go new file mode 100644 index 000000000..4646e9164 --- /dev/null +++ b/arctl/load.go @@ -0,0 +1,137 @@ +/* +Copyright 2023 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "errors" + "os" + + zhipuaiembeddings "github.com/kubeagi/arcadia/pkg/embeddings/zhipuai" + "github.com/kubeagi/arcadia/pkg/llms" + "github.com/kubeagi/arcadia/pkg/llms/zhipuai" + "github.com/kubeagi/arcadia/pkg/vectorstores/chromadb" + "github.com/spf13/cobra" + "github.com/tmc/langchaingo/documentloaders" + "github.com/tmc/langchaingo/embeddings" + openaiEmbeddings "github.com/tmc/langchaingo/embeddings/openai" + "github.com/tmc/langchaingo/schema" + "github.com/tmc/langchaingo/textsplitter" +) + +var ( + embeddingLLMType string + embeddingLLMApiKey string + + document string + language string + + nameSpace string + chunkSize int + chunkOverlap int + + vectorStore string +) + +func NewLoadCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "load [usage]", + Short: "Load documents into VectorStore", + RunE: func(cmd *cobra.Command, args []string) error { + documents, err := LoadAndSplitDocument(context.Background()) + if err != nil { + return err + } + return EmbedDocuments(context.Background(), documents) + }, + } + + cmd.Flags().StringVar(&embeddingLLMType, "embedding-llm-type", string(llms.ZhiPuAI), "llm type to use(Only zhipuai,openai supported now)") + cmd.Flags().StringVar(&embeddingLLMApiKey, "embedding-llm-apikey", "", "apiKey to access embedding service") + + cmd.Flags().StringVar(&vectorStore, "vector-store", "http://localhost:8000", "vector stores to use(Only chroma supported now)") + + cmd.Flags().StringVar(&document, "document", "", "path of the document to load") + cmd.Flags().StringVar(&language, "document-language", "text", "language of the document(Only text,html,csv supported now)") + + cmd.Flags().StringVar(&nameSpace, "namespace", "arcadia", "namespace/collection of the document to load into") + cmd.Flags().IntVar(&chunkSize, "chunk-size", 300, "chunk size for embedding") + cmd.Flags().IntVar(&chunkOverlap, "chunk-overlap", 30, "chunk overlap for embedding") + + cmd.MarkFlagRequired("embedding-llm-apikey") + cmd.MarkFlagRequired("document") + + return cmd +} + +func LoadAndSplitDocument(ctx context.Context) ([]schema.Document, error) { + file, err := os.Open(document) + if err != nil { + return nil, err + } + + var loader documentloaders.Loader + switch language { + case "text": + loader = documentloaders.NewText(file) + case "csv": + loader = documentloaders.NewCSV(file) + case "html": + loader = documentloaders.NewHTML(file) + default: + return nil, errors.New("unsupported document language") + } + + split := textsplitter.NewRecursiveCharacter() + split.ChunkSize = chunkSize + split.ChunkOverlap = chunkOverlap + + return loader.LoadAndSplit(ctx, split) +} + +func EmbedDocuments(ctx context.Context, documents []schema.Document) error { + var embedder embeddings.Embedder + var err error + + switch embeddingLLMType { + case "zhipuai": + embedder, err = zhipuaiembeddings.NewZhiPuAI( + zhipuaiembeddings.WithClient(*zhipuai.NewZhiPuAI(embeddingLLMApiKey)), + ) + if err != nil { + return err + } + case "openai": + embedder, err = openaiEmbeddings.NewOpenAI() + if err != nil { + return err + } + default: + return errors.New("unsupported embedding type") + } + + chroma, err := chromadb.New( + chromadb.WithURL(vectorStore), + chromadb.WithEmbedder(embedder), + chromadb.WithNameSpace(nameSpace), + ) + if err != nil { + return err + } + + return chroma.AddDocuments(ctx, documents) +} diff --git a/arctl/main.go b/arctl/main.go new file mode 100644 index 000000000..f226cdfb8 --- /dev/null +++ b/arctl/main.go @@ -0,0 +1,42 @@ +/* +Copyright 2023 KubeAGI. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" +) + +func NewCLI() *cobra.Command { + arctl := &cobra.Command{ + Use: "arctl [usage]", + Short: "Command line tools for Arcadia", + RunE: func(cmd *cobra.Command, args []string) error { + return nil + }, + } + + arctl.AddCommand(NewLoadCmd()) + arctl.AddCommand(NewChatCmd()) + + return arctl +} + +func main() { + if err := NewCLI().Execute(); err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index b0cac44d1..e7fd9a089 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,11 @@ require ( github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -49,12 +52,15 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/microcosm-cc/bluemonday v1.0.24 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -71,6 +77,7 @@ require ( go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.12.0 // indirect + golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.11.0 // indirect diff --git a/go.sum b/go.sum index e1ac4b4e2..9089ecaf5 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= @@ -72,6 +74,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/amikos-tech/chroma-go v0.0.0-20230901221218-d0087270239e h1:RYla80qSHCeP8WL9Hyjfuk9n/KkDcYwzfaYbfc4cqRQ= github.com/amikos-tech/chroma-go v0.0.0-20230901221218-d0087270239e/go.mod h1:bPy9xmWK59Ix/nteEqIvPfAI0L07di+uZMb+RiYbles= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -79,6 +83,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -270,6 +276,8 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -337,6 +345,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -348,6 +358,8 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -490,6 +502,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= @@ -558,6 +571,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea h1:vLCWI/yYrdEHyN2JzIzPO3aaQJHQdp89IZBA/+azVC4= +golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -584,6 +599,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -632,9 +648,12 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -663,6 +682,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -730,10 +750,14 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -745,6 +769,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -812,6 +837,7 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=