Sqlite (softwerkskammer#1504)

* WIP

* memberstore.js now using sqlitePersistence.js

* groupstore.js now using sqlitePersistence.js

* activitystore.js now using sqlitePersistence.js

* more stores now using sqlitePersistence.js

* save with errors

* all tests up

* remove lots of async/await

* cleaning up

* init sqlite dbname

* init db directory

* more cleanup

* bump deps and small fix

* remove mongo from CI build

* improve migration

* add backup script

* minor improvement in code

.eslintrc

@@ -15,6 +15,7 @@
     "no-array-constructor": 2,
     "no-caller": 2,
     "no-catch-shadow": 2,
+    "no-console": 2,
     "no-eval": 2,
     "no-extend-native": 2,
     "no-extra-bind": 2,

.github/workflows/agora-ci.yml

@@ -10,19 +10,12 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-
-    services:
-      mongodb:
-        image: mongo
-        ports:
-          - 27017:27017
-
     steps:
       - uses: actions/checkout@v4
       - name: Use Node.js v18
         uses: actions/setup-node@v4
         with:
-          node-version: '18.x'
-          cache: "yarn"
+          node-version: 18.x
+          cache: yarn
       - run: yarn install --immutable
       - run: yarn test

CONTRIBUTING.md

@@ -2,13 +2,10 @@
 
 Get in touch with us:
 
-Mailinglist
--------------
+## Mailinglist
 
 Co-ordination via the mailinglist at https://www.softwerkskammer.org/groups/agora
 
-Slack
------
+## Slack
 
 There is an Agora slack channel on the [Software Crafters Slack](https://softwarecrafters.slack.com/messages/agora/). Invite yourself via https://slack.softwarecrafters.org/.
-

Gruntfile.js

@@ -298,8 +298,6 @@ module.exports = function (grunt) {
     },
   });
 
-  process.env.NODE_ICU_DATA = "node_modules/full-icu"; // necessary for timezone stuff
-
   // These plugins provide necessary tasks.
   grunt.loadNpmTasks("grunt-contrib-clean");
   grunt.loadNpmTasks("grunt-contrib-copy");

INSTALL.md

@@ -2,69 +2,19 @@
 
 ## Node.js
 
-It is a node.js project. Therefore you need node.js installed. Get it from [http://nodejs.org](http://nodejs.org).
+It is a node.js project. Therefore, you need node.js installed. Get it from [http://nodejs.org](http://nodejs.org).
 
-You need a current > 8.x version to run the software.
+You need a version compatible with the one defined in package.json to run the software.
 
-Your node.js ships npm in a suitable version.
+## Git
 
-## Additional Software
-
-Some modules are compiled during the installation. Therefore some additional software must be installed.
-
-On Mac OS you can use Homebrew (https://brew.sh/) to install binary dependencies.
-
-On Windows there are currently issues when running the tests:
-* most tests that verify pathnames will fail because of the ` \ ` used by Windows as a separator
-
-### Git
-* Git must be in the system path.
-
-
-    brew install git
-
-### MongoDB
-
-* Install MongoDB (Version 2.4 or greater) [http://www.mongodb.org/downloads](http://www.mongodb.org/downloads)
-* Install and run mongodb (with default settings, i.e. no security - definitely do not do this in production!):
-
-```
-    brew install mongodb
-    sudo mkdir -p /data/db
-    sudo chown 777 /data/db
-    mongod
-```
-
-#### Useful commands
-1. `mongo swk` to connect to the database
-1. `db.memberstore.find().pretty()` to prettry print the content of `memberstore` 
-1. `db.groupstore.update( { _id : ObjectId("5bf9d641b68a0cb25fa4515c") }, { $unset: {contactingOrganizersEnabled: ""} } );` to remove property `contactingOrganizersEnabled` from the group with id `ObjectId("5bf9d641b68a0cb25fa4515c")` 
-
-### Grunt-CLI
-
-We propose to install grunt-cli globally via the -g option of npm. To find out more about this option, see [https://npmjs.org/doc/global.html](https://npmjs.org/doc/global.html).:
-
-* (Optional) To define the installation location of global npm packages on Unix-like Systems create a file called `.npmrc` with the following contents in your Home directory:
-
-        prefix=GLOBALPATH
-        umask=077
-
-* Anywhere, invoke `npm install -g grunt-cli` or `sudo npm install -g grunt-cli` if you don't have sufficient user privileges. You can check the installation with `which grunt`. If a location is returned everything is fine.
-* (Optional) If the directory `GLOBALPATH/bin` is not in your path (you can check with `echo $PATH`), you need to add it to the path: In your Home directory, create or edit the file `.profile` and add the following line:
-
-        export PATH=GLOBALPATH/bin/:$PATH
-
-### Shell for `npm install`
-
-* You need a shell (a.k.a. command line window) that is able to access the stuff you just installed.
+- Git must be in the system path.
 
 ## Preparations for use
 
-* Check out this project into some folder (let's call it *REPO* from now on)
-* Open a shell as described above and go to *REPO*
-* Run `npm install`
-* Start mongodb. If you only plan to use it for development, the default settings are ok. The app as well as the tests use these defaults if you did not change the configuration.
-* (Optional) Instructions for authenticated use are [here](softwerkskammer/lib/persistence/README.md)
+- Check out this project into some folder (let's call it _REPO_ from now on)
+- Open a shell as described above and go to _REPO_
+- Run `yarn install --immutable`
 
 ### The built-in wiki
 
@@ -74,20 +24,12 @@ To set up the built-in wiki follow [these instructions for Softwerkskammer](soft
 
 Configuration for Softwerkskammer:
 
-* Copy the logging configuration file `config-examples/winston-config.json` to `config/winston-config.json`, and adapt the paths if you like.
-* Copy the mailsender configuration file `config-examples/mailsender-config.json` to `config/mailsender-config.json`. Without setting up a proper server sending mails won't work but this configuration is sufficient to be able to start the softwerkskammer app.
-
-
-#### Internationalization for DateTime (Luxon)
-Please see the node section install instructions on https://moment.github.io/luxon/docs/manual/install.html
-
-##### Shortly: 
-Be sure to have node started with either an ENV like ```NODE_ICU_DATA="$(pwd)/node_modules/full-icu"```
-or a startup parameter like ```--icu-data-dir=./node_modules/full-icu```
-
-If you use the run scripts inside WebStorm it is save (they set the parameter where needed).
+- Copy the logging configuration file `config-examples/winston-config.json` to `config/winston-config.json`, and adapt the paths if you like.
+- Copy the mailsender configuration file `config-examples/mailsender-config.json` to `config/mailsender-config.json`. Without setting up a proper server sending mails won't work but this configuration is sufficient to be able to start the softwerkskammer app.
+- Copy the authentication configuration file `config-examples/authentication-config.json` to `config/authentication-config.json`.
 
 #### Mailserver settings
+
 if you want to be able to send mails, you need to configure the mail sender. One way to achieve a running configuration
 is to use a mail server you have access to. The configuration should look like this:
 
@@ -113,31 +55,33 @@ is to use a mail server you have access to. The configuration should look like t
 }
 </code></pre>
 
-### Running the server
+### Running the server - Database Initialization
 
-* Open a shell in *REPO*
-* If your installation is fresh, run the db initialization script:
-   * In any case, run `node softwerkskammer/initialDBSetup`
-* If your installation is fresh or you changed anything:
-   * Run `npm test` (this creates the CSS files and other static contents and performs eslint checking)
+- Open a shell in _REPO_
+- If your installation is fresh, run the db initialization script:
+  - In any case, run `node softwerkskammer/initialDBSetup`
+- If your installation is fresh or you changed anything:
+  - Run `yarn deploy` (this creates the CSS files and other static contents and performs eslint checking)
 
 Now you can start Softwerkskammer:
 
-* `node start-softwerkskammer` - will start the server
-* Now go to your local machine, open a browser and use `http://localhost:17124`
-* The port `17124` is the default and can be changed via the command line option `--port` or via the environment option 'port' to any desired value
-* If your installation is fresh, you should create an account for yourself by registering.
-  * The default setup assumes you are running on localhost for authentication. If you are using a different hostname, you have to edit the configuration file `config-examples/server-config.json`. Follow the instructions in there.
+- `yarn start` - will start the server
+- Now go to your local machine, open a browser and use `http://localhost:17124`
+- The port `17124` is the default and can be changed via the command line option `--port` or via the environment option 'port' to any desired value
+- **Login:** The testdata includes a superuser with email: `[email protected]` password: `testuser`
+- If your installation is fresh, you should create an account for yourself by registering.
+  - The default setup assumes you are running on localhost for authentication. If you are using a different hostname, you have to edit the configuration file `config-examples/server-config.json`. Follow the instructions in there.
 
 ### Full Access to the Applications
 
 Not all features can be accessed without login. Some can only be accessed when you are superuser.
 
 Access for Softwerkskammer:
 
-* Log in to the application. Be aware that Google and Github cannot be used out of the box. Therefore, you should use an OpenID provider such as Stack Exchange, XLogon (https://my.xlogon.net/)
+- Log in to the application. Be aware that GitHub cannot be used out of the box. Therefore, you should use an OpenID provider such as Stack Exchange, XLogon (https://my.xlogon.net/)
   or you can choose one from this list: http://openid.net/get-an-openid/
 
-* To access certain admin features, you may want to become superuser. This step will make you superuser of both applications at once.
-  In order to do this, open `mongo swk`, display all member information via `db.memberstore.find().pretty()` and search for your entry. Select the string after `id`, create a copy of `config-examples/authentication-config.json`
-  and add your id to the `superuser` array.
+- To access certain admin features, you may want to become superuser.
+  This step will make you superuser of both applications at once.
+  In order to do this, start a query on your database file for `memberstore` and search for _your_ entry.
+  Select the `id`, create a copy of `config-examples/authentication-config.json` and add your id to the `superuser` array.

README.md

@@ -1,32 +1,28 @@
-Agora
-=====
+# Agora
 
 [![Build Status](https://github.com/softwerkskammer/Agora/actions/workflows/agora-ci.yml/badge.svg)](https://github.com/softwerkskammer/Agora/actions/workflows/agora-ci.yml)
 
 This is the project to build the new groupware platform for the German Software Craft and Testing communities. It can be seen in action on https://www.softwerkskammer.org
 
 Want to help out? Check out our [Contribution Guidelines](CONTRIBUTING.md).
 
-Installation
-------------
+## Installation
 
 Please refer to our [Installation Guidelines](INSTALL.md) to find out
 how to get Agora running on you local development machine.
 
-Tests
------
+## Tests
 
-Run the tests with `npm test`.
+Run the tests with `yarn test`.
 
 For running specific tests only, use either of these:
+
 - `./node_modules/.bin/grunt karma:once`
 - `./node_modules/.bin/grunt mochacli:test`
 - `./node_modules/.bin/grunt mochacli:testApp`
 - `./node_modules/.bin/grunt mochacli:testWithDB`
 
-
-We are using [JetBrains' WebStorm](http://www.jetbrains.com/webstorm/) as IDE
----------------------------------------------------------------------------
+## We are using [JetBrains' WebStorm](http://www.jetbrains.com/webstorm/) as IDE
 
 [![WebStorm Logo](dev-goodies/icon_WebStorm.png)](http://www.jetbrains.com/webstorm/)
 

backup-sqlite.js

@@ -0,0 +1,40 @@
+/* eslint-disable no-console, no-sync, no-process-exit */
+"use strict";
+
+require("./softwerkskammer/configure.js");
+const Database = require("better-sqlite3");
+const conf = require("simple-configure");
+const path = require("path");
+const AdmZip = require("adm-zip");
+const fs = require("fs");
+const sqlitedb = conf.get("sqlitedb");
+const db = new Database(path.join(__dirname, "softwerkskammer/lib/persistence", sqlitedb));
+const myArgs = process.argv.slice(2);
+const outdir = `${myArgs.length > 0 ? myArgs[0] : ""}`;
+const outpath = path.join(outdir, `backup-db-sqlite-${new Date().toISOString()}.db`);
+
+function zipIt(outfile) {
+  const zip = new AdmZip();
+  zip.addLocalFile(outfile);
+  zip.writeZip(outfile.replace(".db", ".zip"), (err) => {
+    if (err) {
+      console.error("backup failed during zipping:", err);
+      return process.exit(1);
+    }
+    fs.rmSync(outfile);
+  });
+}
+
+if (!fs.existsSync(outdir)) {
+  fs.mkdirSync(outdir);
+}
+
+db.backup(outpath)
+  .then(() => {
+    zipIt(outpath);
+    console.log("backup successful");
+  })
+  .catch((err) => {
+    console.error("backup failed:", err);
+    process.exit(1);
+  });

config-examples/authentication-config.json

@@ -1,6 +1,6 @@
 {
   "githubClientID": "00000000000000000000",
   "githubClientSecret": "0000000000000000000000000000000000000000",
-  "superuser": ["userIDofSuperuser1", "userIDofSuperuser2"],
+  "superuser": ["password:[email protected]"],
   "magicLinkSecret": ""
 }

config-examples/server-config.json

@@ -2,5 +2,6 @@
   "publicUrlPrefix": "http://localhost:17124",
   "secret": "secret",
   "domainname": "softwerkskammer.org",
-  "sessionkey": "softwerkskammer.org"
+  "sessionkey": "softwerkskammer.org",
+  "sqlitedb": "../../../db/database.db"
 }

config/beans.json

@@ -33,23 +33,23 @@
   "memberstore": { "module": "./softwerkskammer/lib/members/memberstore" },
 
   "activitiesPersistence": {
-    "module": "./softwerkskammer/lib/persistence/persistence",
-    "constructorArgs": { "collectionName": "activitystore" }
+    "module": "./softwerkskammer/lib/persistence/sqlitePersistence",
+    "constructorArgs": { "collectionName": "activitystore", "extraColumns": "startDate,endDate,url,version" }
   },
   "activityresultsPersistence": {
-    "module": "./softwerkskammer/lib/persistence/persistence",
+    "module": "./softwerkskammer/lib/persistence/sqlitePersistence",
     "constructorArgs": { "collectionName": "activityresultstore" }
   },
   "groupsPersistence": {
-    "module": "./softwerkskammer/lib/persistence/persistence",
+    "module": "./softwerkskammer/lib/persistence/sqlitePersistence",
     "constructorArgs": { "collectionName": "groupstore" }
   },
   "membersPersistence": {
-    "module": "./softwerkskammer/lib/persistence/persistence",
+    "module": "./softwerkskammer/lib/persistence/sqlitePersistence",
     "constructorArgs": { "collectionName": "memberstore" }
   },
   "settingsPersistence": {
-    "module": "./softwerkskammer/lib/persistence/persistence",
+    "module": "./softwerkskammer/lib/persistence/sqlitePersistence",
     "constructorArgs": { "collectionName": "settingsstore" }
   },