Merge pull request 'Update to new CleanURI (uritools) version' (!83) from cleanuri into master

Reviewed-on: https://gitea.n39.eu/Netz39_Admin/netz39-infra-ansible/pulls/83
Reviewed-by: Alexander Dahl <alex@netz39.de>

holmium.yml

@@ -22,6 +22,7 @@
         hosts:
         - name: gitea.n39.eu
         - name: uritools.n39.eu
+        - name: uritools-api.n39.eu
         - name: sl.n39.eu
         - name: pad.n39.eu
         - name: brotherql-web.n39.eu

inventory.yml

@@ -68,6 +68,19 @@ all:
         3461
     pottwal.n39.eu:
       server_admin: "admin+pottwal@netz39.de"
+
+      # These values need to be set up in RabbitMQ
+      cleanuri_amqp_host: "rabbitmq.n39.eu"
+      cleanuri_amqp_user: "cleanuri"
+      cleanuri_amqp_pass: !vault |
+        $ANSIBLE_VAULT;1.1;AES256
+        36623438333666666263616562386561383564343534383130633336663130323562316337623532
+        6262336637646435666334653834643535316463366132320a393265616331376465383766643539
+        65656130356132333832396266313939326333323161316163653335376266303239663534303731
+        3666383966383463350a616461666534656232666235323333313139353363663763633261346533
+        64623232626531616235386235313763336465366466343334316361396239636337
+      cleanuri_amqp_vhost: "/cleanuri"
+
       shlink_geolite_license_key: !vault |
         $ANSIBLE_VAULT;1.1;AES256
         33626234393039623132663736656363356562383235353737313034613630626339303263366665

pottwal.yml

@@ -8,7 +8,7 @@
     data_dir: "/srv/data"
 
     gitea_host_port: 9091
-    uritools_host_port: 8080
+
     shlink_host_port: 8083
     shlink_domain_name: sl.n39.eu
 
@@ -27,11 +27,19 @@
         dehydrated_domains:
         - name: gitea.n39.eu
         - name: uritools.n39.eu
+        - name: uritools-api.n39.eu
         - name: sl.n39.eu
         - name: pad.n39.eu
     - role: penguineer.dehydrated_cron
     - role: dd24-dyndns-cron
       # variables are set in the inventory
+    - role: cleanuri
+      vars:
+        cleanuri_ui_domain: uritools.n39.eu
+        cleanuri_ui_host_port: 8090
+        cleanuri_api_domain: uritools-api.n39.eu
+        cleanuri_api_host_port: 8091
+        # RabbitMQ setup can be found in the inventory
 
   tasks:
 
@@ -89,25 +97,6 @@
           - 3142:3142
 
 
-    - name: Ensure container for URI tools is running
-      docker_container:
-        name: uritools
-        image: mrtux/clean_uri:0.4.1
-        pull: true
-        state: started
-        detach: yes
-        ports:
-          - "127.0.0.1:{{ uritools_host_port }}:8080"
-        restart_policy: unless-stopped
-
-    - name: Setup proxy site uritools.n39.eu
-      include_role:
-        name: setup-http-site-proxy
-      vars:
-        site_name: uritools.n39.eu
-        proxy_port: "{{ uritools_host_port }}"
-
-
     - name: Ensure container for shlink is running
       docker_container:
         name: shlink

roles/cleanuri/README.md

@@ -0,0 +1,27 @@
+# ansible-role cleanuri
+
+> Set up the [cleanURI](https://github.com/penguineer/cleanURI) service.
+
+## Dependencies
+
+This role uses the [setup-http-site-proxy](../setup-http-site-proxy) role.
+
+## Use
+
+```yaml
+  roles:
+    - role: cleanuri
+      vars:
+        # Make sure to set up HTTPS ingress
+        cleanuri_ui_domain: …
+        cleanuri_ui_host_port: …
+        cleanuri_api_domain: …
+        cleanuri_api_host_port: …
+        # These values need to be set up in RabbitMQ
+        cleanuri_amqp_host: …
+        cleanuri_amqp_user: …
+        cleanuri_amqp_pass: …
+        cleanuri_amqp_vhost: …  # default "/"
+```
+
+Please check [defaults/main.yml](defaults/main.yml) for a complete list of variables.

roles/cleanuri/defaults/main.yml

@@ -0,0 +1,25 @@
+# Defaults for the cleanuri role
+---
+
+# HTTPS ingress
+# cleanuri_ui_domain:
+# cleanuri_ui_host_port:
+# cleanuri_api_domain:
+# cleanuri_api_host_port:
+
+# Credentials for RabbitMQ
+# cleanuri_amqp_host:
+# cleanuri_amqp_user:
+# cleanuri_amqp_pass:
+
+# Change these if they are different in your environment
+cleanuri_amqp_vhost: "/"
+cleanuri_amqp_results: "results"
+cleanuri_amqp_canonizer: "canonizer"
+cleanuri_amqp_retrieval: "extractor"
+
+# Docker images
+cleanuri_image_webui: mrtux/cleanuri-webui:0.1.1
+cleanuri_image_apigateway: mrtux/cleanuri-apigateway:0.3.0
+cleanuri_image_canonizer: mrtux/cleanuri-canonizer:0.3.0
+cleanuri_image_extractor: mrtux/cleanuri-extractor:0.3.0

roles/cleanuri/tasks/main.yml

@@ -0,0 +1,79 @@
+# Tasks for the cleanuri role
+---
+- name: Ensure CleanURI WebUI is running
+  docker_container:
+    name: cleanuri-webui
+    image: "{{ cleanuri_image_webui }}"
+    pull: true
+    state: started
+    detach: yes
+    ports:
+      - "127.0.0.1:{{ cleanuri_ui_host_port }}:80"
+    restart_policy: unless-stopped
+    env:
+      REACT_APP_API_GATEWAY: "https://{{ cleanuri_api_domain }}"
+
+- name: Setup proxy site for the CleanURI WebUI
+  include_role:
+    name: setup-http-site-proxy
+  vars:
+    site_name: "{{ cleanuri_ui_domain }}"
+    proxy_port: "{{ cleanuri_ui_host_port }}"
+
+
+- name: Ensure CleanURI API Gateway is running
+  docker_container:
+    name: cleanuri-apigateway
+    image: "{{ cleanuri_image_apigateway }}"
+    pull: true
+    state: started
+    detach: yes
+    ports:
+      - "127.0.0.1:{{ cleanuri_api_host_port }}:8080"
+    restart_policy: unless-stopped
+    env:
+      AMQP_HOST: "{{ cleanuri_amqp_host }}"
+      AMQP_USER: "{{ cleanuri_amqp_user  }}"
+      AMQP_PASS: "{{ cleanuri_amqp_pass }}"
+      AMQP_VHOST: "{{ cleanuri_amqp_vhost }}"
+      GATEWAY_RESULT_QUEUE: "{{ cleanuri_amqp_results }}"
+      GATEWAY_TASK_RK: "{{ cleanuri_amqp_canonizer }}"
+
+- name: Ensure CleanURI Canonizer is running
+  docker_container:
+    name: cleanuri-canonizer
+    image: "{{ cleanuri_image_canonizer }}"
+    pull: true
+    state: started
+    detach: yes
+    restart_policy: unless-stopped
+    env:
+      AMQP_HOST: "{{ cleanuri_amqp_host }}"
+      AMQP_USER: "{{ cleanuri_amqp_user  }}"
+      AMQP_PASS: "{{ cleanuri_amqp_pass }}"
+      AMQP_VHOST: "{{ cleanuri_amqp_vhost }}"
+      CANONIZER_TASK_QUEUE: "{{ cleanuri_amqp_canonizer }}"
+      EXTRACTOR_TASK_RK: "{{ cleanuri_amqp_retrieval }}"
+
+- name: Ensure CleanURI Extractor is running
+  docker_container:
+    name: cleanuri-extractor
+    image: "{{ cleanuri_image_extractor }}"
+    pull: true
+    state: started
+    detach: yes
+    restart_policy: unless-stopped
+    env:
+      AMQP_HOST: "{{ cleanuri_amqp_host }}"
+      AMQP_USER: "{{ cleanuri_amqp_user  }}"
+      AMQP_PASS: "{{ cleanuri_amqp_pass }}"
+      AMQP_VHOST: "{{ cleanuri_amqp_vhost }}"
+      EXTRACTION_TASK_QUEUE: "{{ cleanuri_amqp_retrieval }}"
+
+
+- name: Setup proxy site the CleanURI API Gateway
+  include_role:
+    name: setup-http-site-proxy
+  vars:
+    site_name: "{{ cleanuri_api_domain }}"
+    proxy_port: "{{ cleanuri_api_host_port }}"