OONI Data, OONI Pipeline split (#60)

Major refactoring of oonidata into two separate
packages:
* oonidata, which is the end user pip installable package to download
and parse measurements (should have minimal dependencies and not require
additional components to run, eg. clickhouse)
* oonipipeline, which is the thing that actually performs the analysis
and processing of data. This does require external dependencies to run.

Eventually we might want to move them into their own respective repos.

This fixes the following issues:
* #41, we keep oonidata as the name
for the CLI tool, but oonipipeline is the actual component doing
analysis.
* #46, we are currently using
temporal.io. After this lands we might want to make prototypes of the
other orchestration systems to compare.
* #57, we might eventually not go for
monorepo, but for the moment this is what we got.

.github/workflows/test_oonidata.yml

@@ -0,0 +1,48 @@
+name: test oonidata
+on: push
+jobs:
+  run_tests:
+    name: ${{ matrix.os }} / ${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}-latest
+    strategy:
+      matrix:
+        os: [Ubuntu]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/datadir/
+          key: oonidata-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/measurements/
+          key: oonidata-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonidata/tests/data/raw_measurements/
+          key: oonidata-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Run all tests
+        run: hatch run cov
+        working-directory: ./oonidata/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonidata
+          working-directory: ./oonidata/

.github/workflows/test_oonipipeline.yml

@@ -0,0 +1,55 @@
+name: test oonipipeline
+on: push
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      # Contrary to oonidata, which is a tool for end users, oonipipeline
+      # targets a single python version to minimize the likelyhood of breakage
+      # in production.
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install hatch
+        run: pip install hatch
+
+      - name: Set up datadir cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/datadir/
+          key: oonipipeline-tests-data-datadir
+
+      - name: Set up measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/measurements/
+          key: oonipipeline-tests-data-measurements
+
+      - name: Set up raw_measurements cache
+        uses: actions/cache@v3
+        with:
+          path: oonipipeline/tests/data/raw_measurements/
+          key: oonipipeline-tests-data-raw_measurements-${{ hashFiles('tests/conftest.py') }}
+
+      - name: Install clickhouse
+        run: |
+          sudo apt-get install -y apt-transport-https ca-certificates dirmngr
+          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
+          echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
+              /etc/apt/sources.list.d/clickhouse.list
+          sudo apt-get update
+          sudo apt-get install -y clickhouse-server clickhouse-client
+
+      - name: Run all tests
+        run: hatch run cov
+        working-directory: ./oonipipeline/
+
+      - name: Upload coverage to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: oonipipeline
+          working-directory: ./oonipipeline/

.gitignore

@@ -1,12 +1,11 @@
 __pycache__
-/.coverage
-/coverage.xml
-/tests/data/datadir/*
-/tests/data/raw_measurements/*
-/tests/data/measurements/*
+.coverage
+.coverage.*
+coverage.xml
+
+.ipynb_checkpoints/
+
 /dist
-/datadir
 /output
 /attic
 /prof
-.ipynb_checkpoints/

.vscode/settings.json

@@ -3,6 +3,7 @@
   "python.testing.pytestArgs": ["tests"],
   "python.testing.unittestEnabled": false,
   "python.testing.pytestEnabled": true,
+  "python.defaultInterpreterPath": "${workspaceFolder}/oonipipeline/.venv",
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"
   }