Merge pull request #24 from claabs/develop

Version 2.0.0

.dockerignore

@@ -1,7 +1,5 @@
-# Ignore everything
-**/*
-
-# Allow files and directories
-!*.js
-!*.json
-!*.ts
+config
+dist
+node_modules
+img
+docs

.eslintrc.js

@@ -6,7 +6,6 @@ module.exports = {
   extends: [
     'airbnb-base',
     'plugin:@typescript-eslint/recommended',
-    'prettier/@typescript-eslint',
     'plugin:prettier/recommended',
   ],
   parser: '@typescript-eslint/parser',
@@ -15,6 +14,12 @@ module.exports = {
     sourceType: 'module'
   },
   plugins: ['@typescript-eslint'],
+  rules: {
+    'import/extensions': 0,
+    'import/prefer-default-export': 0,
+    'no-shadow': 'off',
+    '@typescript-eslint/no-shadow': ['error'],
+  },
   settings: {
     'import/extensions': ['.js', '.ts',],
     'import/parsers': {
@@ -25,5 +30,6 @@ module.exports = {
         extensions: ['.js', '.ts',]
       }
     }
-  }
+  },
+  ignorePatterns: ['dist/**', 'node_modules/**', '.eslintrc.js', 'src/migration/**']
 }

.github/workflows/build-and-push-image.yml

@@ -1,21 +1,27 @@
-name: Build and push image
+name: MultiArchDockerBuild
 
 on:
   push:
     branches:
       - master
+      - develop
 
 jobs:
-  build_and_push_image:
-    name: Build Docker image and push to registries
+  build_multi_arch_image:
+    name: Build multi-arch Docker image.
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
+        with:
+          install: true
 
       - name: Login to DockerHub
         uses: docker/login-action@v1 
@@ -30,15 +36,32 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push
-        id: docker_build_push
+      - name: Build and push master
+        if: ${{ github.ref == 'refs/heads/master' }}
         uses: docker/build-push-action@v2
         with:
+          target: deploy
           push: true
           tags: |
             charlocharlie/markov-discord:latest
             ghcr.io/${{ github.repository }}:latest
-            ghcr.io/${{ github.repository }}:${{ github.sha }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6
+          build-args: |
+            COMMIT_SHA=${{ github.sha }}
+          cache-from: type=gha,scope=${{ github.workflow }}
+          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
 
-      - name: Image digest
-        run: echo ${{ steps.docker_build_push.outputs.digest }}
+      - name: Build and push dev
+        if: ${{ github.ref == 'refs/heads/develop' }}
+        uses: docker/build-push-action@v2
+        with:
+          target: deploy
+          push: true
+          tags: |
+            charlocharlie/markov-discord:dev
+            ghcr.io/claabs/markov-discord:dev
+          platforms: linux/amd64
+          build-args: |
+            COMMIT_SHA=${{ github.sha }}
+          cache-from: type=gha,scope=${{ github.workflow }}
+          cache-to: type=gha,mode=max,scope=${{ github.workflow }}

.github/workflows/typedoc.yml

@@ -0,0 +1,33 @@
+name: Publish Typedoc to Github Pages
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - Readme.md
+      - src/config/classes.ts
+      - .github/workflows/typedoc.yml
+      - package.json
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Node.js for use with actions
+        uses: actions/setup-node@v2
+        with:
+          node-version: 'lts/*'
+          cache: 'npm'
+
+      - name: NPM install
+        run: npm ci
+
+      # Runs a single command using the runners shell
+      - name: Build and lint
+        run: npm run docs
+
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs

.gitignore

@@ -4,6 +4,7 @@ logs
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+docs
 
 # Runtime data
 pids
@@ -65,3 +66,5 @@ config.json
 # error output file
 error.json
 markovDB.json
+
+/config/

.nvmrc

@@ -0,0 +1 @@
+16

.prettierrc.js

@@ -1,5 +1,4 @@
 module.exports = {
     printWidth: 100,
     singleQuote: true,
-    trailingComma: 'es5'
 }

.vscode/launch.json

@@ -13,8 +13,9 @@
                 "ts-node/register"
             ],
             "args": [
-                "${workspaceFolder}/index.ts"
-            ]
+                "${workspaceFolder}/src/index.ts"
+            ],
+            "outputCapture": "std",
         }
     ]
 }

.vscode/settings.json

@@ -13,6 +13,16 @@
     "[typescript]": {
         "editor.formatOnSave": false,
     },
-    "eslint.enable": true,
-    "typescript.tsdk": "node_modules\\typescript\\lib",
+    "[json]": {
+        "files.insertFinalNewline": true
+    },
+    "typescript.tsdk": "node_modules/typescript/lib",
+    "sqltools.connections": [
+        {
+            "previewLimit": 50,
+            "driver": "SQLite",
+            "name": "Local SQLite",
+            "database": "./config/db/db.sqlite3"
+        }
+    ],
 }

CHANGELOG.md

@@ -0,0 +1,101 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+## Versions
+
+### 2.0.0
+
+#### Breaking Changes
+
+* Config option `prefix` renamed to `messageCommandPrefix`
+* Config option `game` renamed to `activity`
+* Config option `role` renamed to `userRoleIds`. Changed from string to array of strings.
+* Docker internal volume path moved from `/usr/src/markbot/config` to `/usr/app/config`
+* Database changed from JSON files to a SQLite database. You'll need to re-train the bot to use it again.
+* The bot must be explicitly granted permission to listen to a list of channels before using it. Configure it with `/listen`.
+* Docker user changed from `root` to `node`. You may need to update your mounted volume's permissions.
+* pm2 has been removed from the Docker container. Make sure to add `--restart=unless-stopped` to your Docker run config to ensure the same resiliency.
+
+#### New Features
+
+* Data is stored in a relational database to reduce memory and disk read/write usage, as well as to decrease latency
+* The bot can be restricted to only learn/listen from a strict list of channels
+* Bot responses can be seeded by a short phrase
+* Discord slash command support
+* Discord thread support
+* Many new config options available at <https://claabs.github.io/markov-discord/classes/AppConfig.html>
+  * Owner IDs
+  * Log level
+  * Slash command name
+* Config file supports [JSON5](https://json5.org/) (comments, trailing commas, etc). It also may use the `.json5` file extension if you prefer.
+* Generated responses will now never ping a user or role, only just highlight their name
+
+### 0.7.3
+
+* Fix crash when fetched messages is empty
+* Update docs
+* Update dependencies
+
+### 0.7.2
+
+* Fix @everyone replacement
+
+### 0.7.1
+
+* Readme updates
+* Config loading fix
+* Fix min score
+* Add generator options to config
+* Document Node 12 update
+
+### 0.7.0
+
+* Convert project to Typescript
+* Optimize Docker build (smaller image)
+* Load corpus from filesystem to reduce memory load
+
+### 0.6.2
+
+* Fix MarkovDB not loading on boot
+
+### 0.6.1
+
+* Fix bot crashing on scheduled regen
+
+### 0.6.0
+
+* Added Docker deploy functionality.
+* Moved config and database to `./config` directory. Existing configs will be migrated.
+* Config-less support via bot token located in an environment variable.
+* Update dependencies.
+* Change corpus regen time to 4 AM.
+
+### 0.5.0
+
+* Fixed bug where `!mark help` didn't work.
+* Only admins can train.
+* The bot responds when mentioned.
+* The bot cannot mention @everyone.
+* Added version number to help.
+* Added `!mark tts` for a quieter TTS response.
+* Readme overhaul.
+* Simpler config loading.
+
+### 0.4.0
+
+* Huge refactor.
+* Added `!mark debug` which sends debug info alongside the message.
+* Converted the fetchMessages function to async/await (updating the requirement to Node.js 8).
+* Updated module versions.
+* Added faster unique-array-by-property function
+* Added linting and linted the project.
+
+### 0.3.0
+
+* Added TTS support and random message attachments.
+* Deleted messages no longer persist in the database longer than 24 hours.
+
+### 0.2.0
+
+* Updated training algorithm and data structure.

Dockerfile

@@ -1,49 +1,55 @@
 ########
 # BASE
 ########
-FROM keymetrics/pm2:12-alpine as base
-WORKDIR /usr/src/markbot
+FROM node:16-alpine3.15 as base
 
-COPY package*.json ./
-# Install build tools for erlpack, then install prod deps only, then remove build tools
-RUN apk add --no-cache make gcc g++ python && \
-    npm ci --only=production && \
-    apk del make gcc g++ python
+WORKDIR /usr/app
+
+RUN apk add --no-cache tini
 
 ########
 # BUILD
 ########
 FROM base as build
 
-# Copy all *.json, *.js, *.ts
-COPY . .
-# Prod deps already installed, add dev deps
-RUN npm i
+COPY package*.json ./
+# Install build tools for erlpack, then install prod deps only
+RUN apk add --no-cache make gcc g++ python3 \
+    && npm ci --only=production
+
+# Copy all jsons
+COPY package*.json tsconfig.json ./
+
+# Add dev deps
+RUN npm ci
+
+# Copy source code
+COPY src src
 
 RUN npm run build
 
 ########
 # DEPLOY
 ########
-FROM keymetrics/pm2:12-alpine as deploy
-WORKDIR /usr/src/markbot
+FROM base as deploy
 
-ENV NPM_CONFIG_LOGLEVEL warn
+USER node
 
 # Steal node_modules from base image
-COPY --from=base /usr/src/markbot/node_modules ./node_modules/
+COPY --from=build /usr/app/node_modules node_modules
 
 # Steal compiled code from build image
-COPY --from=build /usr/src/markbot/dist ./
+COPY --from=build /usr/app/dist dist
 
 # Copy package.json for version number
-COPY package*.json ./
+COPY package*.json ormconfig.js ./
 
-# Copy PM2 config
-COPY ecosystem.config.js .
+# RUN mkdir config
 
-RUN mkdir config
+ARG COMMIT_SHA=""
 
-# RUN ls -al
+ENV NODE_ENV=production \
+    COMMIT_SHA=${COMMIT_SHA}
 
-CMD [ "pm2-runtime", "start", "ecosystem.config.js" ]
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD [ "node", "/usr/app/dist/index.js" ]