Merge branch 'dm2' into command-pattern

.github/workflows/continous-integration.yml

@@ -519,8 +519,8 @@ jobs:
           path: |
             ${{ github.workspace }}/${{ matrix.test }}-coverage
 
-  upload_coverage_reports:
-    name: Upload coverage reports to codeclimate
+  prepare_coverage_reports:
+    name: Prepare coverage reports
     if: github.ref_type != 'tag'
     runs-on: ubuntu-22.04
     # Always upload results even if tests failed
@@ -561,15 +561,34 @@ jobs:
           coverage combine "${final_dir}/"*
           coverage xml
 
-      - name: Upload reports to codeclimate
+  sonarcloud:
+    name: SonarCloud Scan
+    if: github.ref_type != 'tag'
+    runs-on: ubuntu-22.04
+    needs:
+      - prepare_coverage_reports
+
+    steps:
+      - name: Checkout git repository 🕝
         if: needs.changes.outputs.backend == 'true'
-        uses: paambaati/codeclimate-action@b649ad206d2e83dafb9ed130deba698aa1b41d78
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
+        with:
+          # Disabling shallow clone is recommended for improving relevancy of coverage reporting
+          fetch-depth: 0
+
+      - name: Analyse code with SonarCloud
+        uses: sonarsource/sonarcloud-github-action@5875562561d22a34be0c657405578705a169af6c
         env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CODECLIMATE_REPORTER_ID }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
         with:
-          coverageLocations: |
-            ${{ github.workspace }}/coverage.xml:coverage.py
-          debug: true
+          args: >
+            -Dsonar.organization=rasahq
+            -Dsonar.projectKey=RasaHQ_rasa
+            -Dsonar.sources=.
+            -Dsonar.python.coverage.reportPaths=tests_coverage/coverage.xml
+            -Dsonar.host.url=https://sonarcloud.io
+            -Dsonar.verbose=true
 
   integration_test:
     name: Run Non-Sequential Integration Tests

.github/workflows/sonar-project.properties

@@ -0,0 +1,10 @@
+# Organization and project keys are displayed in the right sidebar of the project homepage
+sonar.organization=rasahq
+sonar.projectKey=RasaHQ_rasa
+sonar.host.url=https://sonarcloud.io/project/overview?id=RasaHQ_rasa
+
+sonar.python.coverage.reportPaths=./tests_coverage/coverage.xml
+
+# relative paths to source directories. More details and properties are described
+sonar.tests=./tests/
+sonar.verbose=true

README.md

@@ -6,7 +6,7 @@
 [![PyPI version](https://badge.fury.io/py/rasa.svg)](https://badge.fury.io/py/rasa)
 [![Supported Python Versions](https://img.shields.io/pypi/pyversions/rasa.svg)](https://pypi.python.org/pypi/rasa)
 [![Build Status](https://github.com/RasaHQ/rasa/workflows/Continuous%20Integration/badge.svg)](https://github.com/RasaHQ/rasa/actions)
-[![Coverage Status](https://api.codeclimate.com/v1/badges/756dc6fea1d5d3e127f7/test_coverage)](https://codeclimate.com/github/RasaHQ/rasa/)
+[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=RasaHQ_rasa&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=RasaHQ_rasa)
 [![Documentation Status](https://img.shields.io/badge/docs-stable-brightgreen.svg)](https://rasa.com/docs)
 ![Documentation Build](https://img.shields.io/netlify/d2e447e4-5a5e-4dc7-be5d-7c04ae7ff706?label=Documentation%20Build)
 [![FOSSA Status](https://app.fossa.com/api/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git.svg?type=shield)](https://app.fossa.com/projects/custom%2B8141%2Fgit%40github.com%3ARasaHQ%2Frasa.git?ref=badge_shield)

changelog/12637.improvement.md

@@ -0,0 +1 @@
+Added a human readable component to structlog using the `event_info` key and made it the default rendered key if present.

changelog/12638.bugfix.md

@@ -0,0 +1 @@
+Fix the issue with the most recent model not being selected if the owner or permissions where modified on the model file.

pyproject.toml

@@ -102,7 +102,7 @@ fbmessenger = "~6.0.0"
 pykwalify = ">=1.7,<1.9"
 coloredlogs = ">=10,<16"
 "ruamel.yaml" = ">=0.16.5,<0.17.22"
-pyyaml = ">=5.3.1,<6.0"
+pyyaml = ">=6.0"
 twilio = ">=6.26,<8.4"
 webexteamssdk = ">=1.1.1,<1.7.0"
 mattermostwrapper = "~2.2"

rasa/core/actions/forms.py

@@ -615,7 +615,8 @@ async def activate(
             logger.debug("No pre-filled required slots to validate.")
         else:
             structlogger.debug(
-                "forms.activate.form", prefilled_slots=copy.deepcopy(prefilled_slots)
+                "forms.validate.prefilled_slots",
+                prefilled_slots=copy.deepcopy(prefilled_slots),
             )
 
         validate_name = f"validate_{self.name()}"

rasa/core/channels/facebook.py

@@ -128,7 +128,7 @@ async def message(
             text = attachment["payload"]["url"]
         else:
             structlogger.warning(
-                "facebook.message.handle", message=copy.deepcopy(message)
+                "facebook.message.cannot.handle", message=copy.deepcopy(message)
             )
             return
 

rasa/core/channels/hangouts.py

@@ -323,7 +323,7 @@ async def receive(request: Request) -> HTTPResponse:
                 )
             except CancelledError:
                 structlogger.error(
-                    "hangouts.message.blueprint", text=copy.deepcopy(text)
+                    "hangouts.message.blueprint.timeout", text=copy.deepcopy(text)
                 )
             except Exception:
                 structlogger.exception(

rasa/core/channels/rest.py

@@ -170,7 +170,7 @@ async def receive(request: Request) -> Union[ResponseStream, HTTPResponse]:
                     )
                 except CancelledError:
                     structlogger.error(
-                        "rest.message.received", text=copy.deepcopy(text)
+                        "rest.message.received.timeout", text=copy.deepcopy(text)
                     )
                 except Exception:
                     structlogger.exception(

rasa/core/http_interpreter.py

@@ -50,6 +50,7 @@ async def _rasa_http_parse(
             structlogger.error(
                 "http.parse.text",
                 text=copy.deepcopy(text),
+                event_info="No rasa NLU server specified!",
             )
             return None
 

rasa/core/nlg/interpolator.py

@@ -38,10 +38,16 @@ def interpolate_text(response: Text, values: Dict[Text, Text]) -> Text:
 
         return text
     except KeyError as e:
+        event_info = (
+            "The specified slot name does not exist, "
+            "and no explicit value was provided during the response invocation. "
+            "Return the response without populating it."
+        )
         structlogger.exception(
             "interpolator.interpolate.text",
             response=copy.deepcopy(response),
             placeholder_key=e.args[0],
+            event_info=event_info,
         )
         return response
 

rasa/model.py

@@ -70,7 +70,7 @@ def get_latest_model(model_path: Text = DEFAULT_MODELS_PATH) -> Optional[Text]:
     if len(list_of_files) == 0:
         return None
 
-    return max(list_of_files, key=os.path.getctime)
+    return max(list_of_files, key=os.path.getmtime)
 
 
 def get_model_for_finetuning(

rasa/shared/utils/io.py

@@ -273,6 +273,7 @@ def get_list_fingerprint(
 
 def get_text_hash(text: Text, encoding: Text = DEFAULT_ENCODING) -> Text:
     """Calculate the md5 hash for a text."""
+    # deepcode ignore InsecureHash: Not used for a cryptographic purpose
     return md5(text.encode(encoding)).hexdigest()  # nosec
 
 

rasa/utils/log_utils.py

@@ -6,13 +6,28 @@
 
 import structlog
 from structlog_sentry import SentryProcessor
+from structlog.dev import ConsoleRenderer
+from structlog.typing import EventDict, WrappedLogger
 from rasa.shared.constants import ENV_LOG_LEVEL, DEFAULT_LOG_LEVEL
 from rasa.plugin import plugin_manager
 
 
 FORCE_JSON_LOGGING = os.environ.get("FORCE_JSON_LOGGING")
 
 
+class HumanConsoleRenderer(ConsoleRenderer):
+    """Console renderer that outputs human-readable logs."""
+
+    def __call__(self, logger: WrappedLogger, name: str, event_dict: EventDict) -> str:
+        if "event_info" in event_dict:
+            event_key = event_dict["event"]
+            event_dict["event"] = event_dict["event_info"]
+            event_dict["event_key"] = event_key
+            del event_dict["event_info"]
+
+        return super().__call__(logger, name, event_dict)
+
+
 def _anonymizer(
     _: structlog.BoundLogger, __: str, event_dict: Dict[str, Any]
 ) -> Dict[str, Any]:
@@ -43,7 +58,6 @@ def _anonymizer(
         "entity",
         "token_text",
         "user_message",
-        "activity",
         "json_message",
     ]
     anonymization_pipeline = plugin_manager().hook.get_anonymization_pipeline()
@@ -98,7 +112,7 @@ def configure_structlog(
         # Pretty printing when we run in a terminal session.
         # Automatically prints pretty tracebacks when "rich" is installed
         processors = shared_processors + [
-            structlog.dev.ConsoleRenderer(),
+            HumanConsoleRenderer(),
         ]
     else:
         # Print JSON when we run, e.g., in a Docker container.

tests/core/channels/test_facebook.py

@@ -13,6 +13,7 @@ def test_facebook_channel():
     input_channel = FacebookInput(
         fb_verify="YOUR_FB_VERIFY",
         # you need tell facebook this token, to confirm your URL
+        # deepcode ignore HardcodedNonCryptoSecret/test: Test credential
         fb_secret="YOUR_FB_SECRET",  # your app secret
         fb_access_token="YOUR_FB_PAGE_ACCESS_TOKEN"
         # token for the page you subscribed to

tests/core/channels/test_slack.py

@@ -1,3 +1,6 @@
+# file deepcode ignore HardcodedNonCryptoSecret/test: Secrets are all just examples for tests. # noqa: E501
+# file deepcode ignore NoHardcodedCredentials/test: Secrets are all just examples for tests. # noqa: E501
+
 from http import HTTPStatus
 import json
 import logging

tests/core/channels/test_telegram.py

@@ -1,3 +1,6 @@
+# file deepcode ignore HardcodedNonCryptoSecret/test: Secrets are all just examples for tests. # noqa: E501
+# file deepcode ignore NoHardcodedCredentials/test: Secrets are all just examples for tests. # noqa: E501
+
 import json
 import logging
 from unittest.mock import patch

tests/core/channels/test_twilio.py

@@ -10,6 +10,7 @@ def test_twilio_channel():
     input_channel = TwilioInput(
         account_sid="ACCOUNT_SID",
         # Find your Account SID and Auth Token at twilio.com/console
+        # deepcode ignore HardcodedNonCryptoSecret/test: Test credential
         auth_token="AUTH_TOKEN",
         # Phone Number you want to use
         twilio_number="TWILIO_NUMBER",

tests/core/test_broker.py

@@ -404,7 +404,9 @@ async def test_sql_connection_error(monkeypatch: MonkeyPatch):
 def test_pika_event_broker_configure_url(
     host: Text, expected_url: Optional[Text]
 ) -> None:
+    # deepcode ignore NoHardcodedCredentials/test: Test credential
     username = "test_user"
+    # deepcode ignore NoHardcodedPasswords/test: Test credential
     password = "test_pass"
     broker = PikaEventBroker(host=host, username=username, password=password)
     url = broker._configure_url()

tests/core/test_channels.py

@@ -1,3 +1,5 @@
+# file deepcode ignore HardcodedNonCryptoSecret/test: Secrets are all just examples for tests. # noqa: E501
+
 import logging
 
 import jwt

tests/core/test_tracker_stores.py

@@ -1,3 +1,5 @@
+# file deepcode ignore NoHardcodedCredentials/test: Secrets are all just examples for tests. # noqa: E501
+
 import logging
 import warnings
 from collections import deque
@@ -397,6 +399,7 @@ def test_sql_tracker_store_logs_do_not_show_password(caplog: LogCaptureFixture):
     port = 9901
     db = "some-database"
     username = "db-user"
+    # deepcode ignore NoHardcodedPasswords/test: Test credential
     password = "some-password"
 
     with caplog.at_level(logging.DEBUG):

tests/integration_tests/core/brokers/test_pika.py

@@ -1,3 +1,5 @@
+# file deepcode ignore NoHardcodedCredentials/test: Secrets are all just examples for tests. # noqa: E501
+
 from typing import Text
 
 import docker
@@ -109,6 +111,7 @@ async def test_pika_event_broker_connect_with_path_and_query_params_in_url(
     host_component: Text,
 ) -> None:
     username = "myuser"
+    # deepcode ignore NoHardcodedPasswords/test: Test credential
     password = "mypassword"
     vhost = "myvhost"
     hostname = "my-rabbitmq"

tests/integration_tests/core/test_agent.py

@@ -35,6 +35,7 @@ def aws_endpoint_url() -> Text:
 def create_user_with_access_key_and_attached_policy(region_name: Text) -> Any:
     """Create a user and an access key for them."""
     client = boto3.client("iam", region_name=region_name)
+    # deepcode ignore NoHardcodedCredentials/test: Test credential
     client.create_user(UserName="test_user")
 
     policy_document = {

tests/test_server.py

@@ -1,3 +1,5 @@
+# file deepcode ignore HardcodedNonCryptoSecret/test: Secrets are all just examples for tests. # noqa: E501
+
 import asyncio
 import json
 import os