merge with stg

LinkedInLearning · Feb 24, 2024 · 7831d5e · 7831d5e
2 parents 6b15f5d + 4c9fb94
commit 7831d5e
Show file tree

Hide file tree

Showing 156 changed files with 465,094 additions and 55 deletions.
diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Dockerfile.dev
@@ -1,11 +1,11 @@
 # Setting an R environment from scratch 
 # Step 1 - Import base image
-FROM arm64v8/ubuntu:22.04
+FROM ubuntu:22.04
 
 # Step 2 - Set arguments and environment variables
 # Define arguments
-ARG PROJECT_NAME=PROJECT_NAME
-ARG VENV_NAME=VENV_NAME
+ARG PROJECT_NAME="EIA_DATA_PIPELINE"
+ARG VENV_NAME="LINKEDIN_LEARNING"
 ARG R_VERSION_MAJOR=4
 ARG R_VERSION_MINOR=3
 ARG R_VERSION_PATCH=1
@@ -92,14 +92,13 @@ RUN Rscript ./settings/install_packages.R
 
 # Installing Quarto
 RUN bash ./settings/install_quarto.sh $QUARTO_VER
-COPY .Rprofile /root/
 
 # Step 5 - Set Python Environment and install radian
-RUN python3 -m venv /opt/$VENV_NAME  \
-    && export PATH=/opt/$VENV_NAME/bin:$PATH \
-    && echo "source /opt/$VENV_NAME/bin/activate" >> ~/.bashrc
+COPY install_requirements.sh ./settings/
+RUN bash ./settings/install_requirements.sh $VENV_NAME
 
-RUN pip3 install -r ./settings/requirements.txt
+COPY .Rprofile /
+RUN echo "alias r='radian --profile=/.Rprofile'" >> ~/.bashrc
 
 
 

diff --git a/.devcontainer/Dockerfile.devc b/.devcontainer/Dockerfile.devc
@@ -0,0 +1,4 @@
+FROM "docker.io/rkrispin/eia_data_refresh:dev.0.0.0.9000"
+
+COPY .Rprofile /
+RUN echo "alias r='radian --profile=/.Rprofile'" >> ~/.bashrc
diff --git a/.devcontainer/build_docker.sh b/.devcontainer/build_docker.sh
@@ -5,18 +5,18 @@ echo "Build the docker"
 docker build . -f Dockerfile.dev \
                --progress=plain \
                --build-arg PROJECT_NAME="EIA Data Automation" \
-               --build-arg VENV_NAME="R_ENV" \
+               --build-arg VENV_NAME="LINKEDIN_LEARNING" \
                --build-arg R_VERSION_MAJOR=4 \
                --build-arg R_VERSION_MINOR=3 \
                --build-arg R_VERSION_PATCH=1 \
-                --build-arg DEBIAN_FRONTEND=noninteractive \
-                 --build-arg CRAN_MIRROR="https://cran.rstudio.com/" \
-               --build-arg QUARTO_VER="1.3.450" \
-               -t rkrispin/eia_data_refresh:arm64.0.0.0.9000
+               --build-arg DEBIAN_FRONTEND=noninteractive \
+                --build-arg CRAN_MIRROR="https://cran.rstudio.com/" \
+               --build-arg QUARTO_VER=$QUARTO_VER \
+               -t rkrispin/eia_data_refresh:0.0.0.9004
 
 if [[ $? = 0 ]] ; then
 echo "Pushing docker..."
-docker push rkrispin/eia_data_refresh:arm64.0.0.0.9000
+docker push rkrispin/eia_data_refresh:0.0.0.9004
 else
 echo "Docker build failed"
 fi
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,23 +1,13 @@
 {
     "name": "EIA Data Automation",
     // "build": {
-    //     "dockerfile": "Dockerfile.dev",
+    //     "dockerfile": "Dockerfile.devc",
     //     "context": ".",
-    //     "args": {
-    //         "PROJECT_NAME": "EIA-Data-Automation",
-    //         "VENV_NAME": "R_ENV",
-    //         "R_VERSION_MAJOR": "4",
-    //         "R_VERSION_MINOR": "3",
-    //         "R_VERSION_PATCH": "1",
-    //         "CRAN_MIRROR": "https://cran.rstudio.com/",
-    //         "QUARTO_VER": "1.3.450"
-    //     }
     // },
-    "image": "docker.io/rkrispin/eia_data_refresh:dev.0.0.0.9000",
+    "image": "docker.io/rkrispin/eia_data_refresh:arm64.0.0.0.9003",
     "settings": {
-        "files.associations": {
-            "*.Rmd": "rmd"
-        }
+        "python.defaultInterpreterPath": "/opt/R_ENV/bin/python3",
+        "python.selectInterpreter": "/opt/R_ENV/bin/python3"
     },
     "customizations": {
         "vscode": {
@@ -46,6 +36,5 @@
     },
     "remoteEnv": {
         "EIA_API_KEY": "${localEnv:EIA_API_KEY}"
-    },
-    "postCreateCommand": "radian"
+    }
 }
diff --git a/.devcontainer/install_requirements.sh b/.devcontainer/install_requirements.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+VENV_NAME=$1
+
+python3 -m venv /opt/$VENV_NAME  \
+    && export PATH=/opt/$VENV_NAME/bin:$PATH \
+    && echo "source /opt/$VENV_NAME/bin/activate" >> ~/.bashrc
+
+apt-get update && apt-get install -y --no-install-recommends \
+    gcc \
+    python3-dev \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+source /opt/$VENV_NAME/bin/activate 
+
+pip install --upgrade pip
+
+pip3 install  --no-cache-dir -r ./settings/requirements.txt
diff --git a/.devcontainer/packages.json b/.devcontainer/packages.json
@@ -2,57 +2,61 @@
     "packages": [
         {
             "package": "BH",
-            "version":"1.81.0-1"
+            "version": "1.81.0-1"
         },
         {
             "package": "languageserver",
-            "version":"0.3.12"
+            "version": "0.3.12"
         },
         {
             "package": "httpgd",
-            "version":"1.3.1"
+            "version": "1.3.1"
         },
         {
             "package": "rmarkdown",
-            "version":"2.21"
+            "version": "2.21"
         },
         {
             "package": "dplyr",
-            "version":"1.1.3"
+            "version": "1.1.3"
         },
         {
             "package": "ggplot2",
-            "version":"3.4.3"
+            "version": "3.4.3"
         },
         {
             "package": "shiny",
-            "version":"1.7.4"
+            "version": "1.7.4"
         },
         {
             "package": "plotly",
-            "version":"4.10.2"
+            "version": "4.10.2"
         },
         {
             "package": "EIAapi",
-            "version":"0.1.2"
+            "version": "0.1.2"
         },
         {
             "package": "devtools",
-            "version":"2.4.5"
+            "version": "2.4.5"
         },
         {
             "package": "readr",
-            "version":"2.1.4"
+            "version": "2.1.4"
+        },
+        {
+            "package": "gt",
+            "version": "0.10.1"
         }
     ],
     "package_dev": [
         {
             "package": "BH",
-            "version":"1.81.0-1"
+            "version": "1.81.0-1"
         },
         {
             "package": "httpgd",
-            "version":"1.3.1"
+            "version": "1.3.1"
         }
     ]
 }
diff --git a/.devcontainer/requirements.txt b/.devcontainer/requirements.txt
@@ -1 +1,11 @@
-radian
+wheel
+radian
+pandas
+numpy
+gensim
+ipykernel
+requests
+plotly
+ipywidgets
+jupyter
+great_tables
diff --git a/.github/workflows/data_refresh.yml b/.github/workflows/data_refresh.yml
@@ -7,7 +7,7 @@ jobs:
   refresh-the-dashboard:
     runs-on: ubuntu-22.04
     container:
-      image: docker.io/rkrispin/eia_data_refresh:0.0.0.9004
+      image: docker.io/rkrispin/eia_data_refresh:dev.0.0.0.9004
     steps:
       - name: checkout_repo
         uses: actions/checkout@v3

diff --git a/.github/workflows/data_refresh_R.yml b/.github/workflows/data_refresh_R.yml
@@ -0,0 +1,19 @@
+name: Data Refresh - R (Docker)
+
+on:
+  schedule:
+    - cron: "0 */1 * * *"
+jobs:
+  refresh-the-dashboard:
+    runs-on: ubuntu-22.04
+    container:
+      image: docker.io/rkrispin/eia_data_refresh:0.0.0.9004
+    steps:
+      - name: checkout_repo
+        uses: actions/checkout@v3
+        with:
+          ref: "stg"
+      - name: Data Refresh
+        run: bash ./R/data_refresh_R.sh "RamiKrispin" "[email protected]"
+        env:
+          EIA_API_KEY: ${{ secrets.EIA_API_KEY }}
diff --git a/.github/workflows/data_refresh_py.yml b/.github/workflows/data_refresh_py.yml
@@ -16,4 +16,4 @@ jobs:
       - name: Data Refresh
         run: bash ./python/data_refresh_py.sh "RamiKrispin" "[email protected]"
         env:
-          API_KEY: ${{ secrets.EIA_API_KEY }}
+          EIA_API_KEY: ${{ secrets.EIA_API_KEY }}