From 24929a36bcd0462f7bd969646da63a2c3b8f4e70 Mon Sep 17 00:00:00 2001 From: Stefan Haun Date: Thu, 15 Sep 2022 17:04:21 +0200 Subject: [PATCH 1/5] :sparkles: Add a role to set up cleanuri (uritools) --- roles/cleanuri/README.md | 27 +++++++++++ roles/cleanuri/defaults/main.yml | 25 ++++++++++ roles/cleanuri/tasks/main.yml | 79 ++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 roles/cleanuri/README.md create mode 100644 roles/cleanuri/defaults/main.yml create mode 100644 roles/cleanuri/tasks/main.yml diff --git a/roles/cleanuri/README.md b/roles/cleanuri/README.md new file mode 100644 index 0000000..9ea7a75 --- /dev/null +++ b/roles/cleanuri/README.md @@ -0,0 +1,27 @@ +# ansible-role cleanuri + +> Set up the [cleanURI](https://github.com/penguineer/cleanURI) service. + +## Dependencies + +This role uses the [setup-http-site-proxy](../setup-http-site-proxy) role. + +## Use + +```yaml + roles: + - role: cleanuri + vars: + # Make sure to set up HTTPS ingress + cleanuri_ui_domain: … + cleanuri_ui_host_port: … + cleanuri_api_domain: … + cleanuri_api_host_port: … + # These values need to be set up in RabbitMQ + cleanuri_amqp_host: … + cleanuri_amqp_user: … + cleanuri_amqp_pass: … + cleanuri_amqp_vhost: … # default "/" +``` + +Please check [defaults/main.yml](defaults/main.yml) for a complete list of variables. diff --git a/roles/cleanuri/defaults/main.yml b/roles/cleanuri/defaults/main.yml new file mode 100644 index 0000000..a8602f3 --- /dev/null +++ b/roles/cleanuri/defaults/main.yml @@ -0,0 +1,25 @@ +# Defaults for the cleanuri role +--- + +# HTTPS ingress +# cleanuri_ui_domain: +# cleanuri_ui_host_port: +# cleanuri_api_domain: +# cleanuri_api_host_port: + +# Credentials for RabbitMQ +# cleanuri_amqp_host: +# cleanuri_amqp_user: +# cleanuri_amqp_pass: + +# Change these if they are different in your environment +cleanuri_amqp_vhost: "/" +cleanuri_amqp_results: "results" +cleanuri_amqp_canonizer: "canonizer" +cleanuri_amqp_retrieval: "extractor" + +# Docker images +cleanuri_image_webui: mrtux/cleanuri-webui:0.1.1 +cleanuri_image_apigateway: mrtux/cleanuri-apigateway:0.3.0 +cleanuri_image_canonizer: mrtux/cleanuri-canonizer:0.3.0 +cleanuri_image_extractor: mrtux/cleanuri-extractor:0.3.0 diff --git a/roles/cleanuri/tasks/main.yml b/roles/cleanuri/tasks/main.yml new file mode 100644 index 0000000..60c414e --- /dev/null +++ b/roles/cleanuri/tasks/main.yml @@ -0,0 +1,79 @@ +# Tasks for the cleanuri role +--- +- name: Ensure CleanURI WebUI is running + docker_container: + name: cleanuri-webui + image: "{{ cleanuri_image_webui }}" + pull: true + state: started + detach: yes + ports: + - "127.0.0.1:{{ cleanuri_ui_host_port }}:80" + restart_policy: unless-stopped + env: + REACT_APP_API_GATEWAY: "https://{{ cleanuri_api_domain }}" + +- name: Setup proxy site for the CleanURI WebUI + include_role: + name: setup-http-site-proxy + vars: + site_name: "{{ cleanuri_ui_domain }}" + proxy_port: "{{ cleanuri_ui_host_port }}" + + +- name: Ensure CleanURI API Gateway is running + docker_container: + name: cleanuri-apigateway + image: "{{ cleanuri_image_apigateway }}" + pull: true + state: started + detach: yes + ports: + - "127.0.0.1:{{ cleanuri_api_host_port }}:8080" + restart_policy: unless-stopped + env: + AMQP_HOST: "{{ cleanuri_amqp_host }}" + AMQP_USER: "{{ cleanuri_amqp_user }}" + AMQP_PASS: "{{ cleanuri_amqp_pass }}" + AMQP_VHOST: "{{ cleanuri_amqp_vhost }}" + GATEWAY_RESULT_QUEUE: "{{ cleanuri_amqp_results }}" + GATEWAY_TASK_RK: "{{ cleanuri_amqp_canonizer }}" + +- name: Ensure CleanURI Canonizer is running + docker_container: + name: cleanuri-canonizer + image: "{{ cleanuri_image_canonizer }}" + pull: true + state: started + detach: yes + restart_policy: unless-stopped + env: + AMQP_HOST: "{{ cleanuri_amqp_host }}" + AMQP_USER: "{{ cleanuri_amqp_user }}" + AMQP_PASS: "{{ cleanuri_amqp_pass }}" + AMQP_VHOST: "{{ cleanuri_amqp_vhost }}" + CANONIZER_TASK_QUEUE: "{{ cleanuri_amqp_canonizer }}" + EXTRACTOR_TASK_RK: "{{ cleanuri_amqp_retrieval }}" + +- name: Ensure CleanURI Extractor is running + docker_container: + name: cleanuri-extractor + image: "{{ cleanuri_image_extractor }}" + pull: true + state: started + detach: yes + restart_policy: unless-stopped + env: + AMQP_HOST: "{{ cleanuri_amqp_host }}" + AMQP_USER: "{{ cleanuri_amqp_user }}" + AMQP_PASS: "{{ cleanuri_amqp_pass }}" + AMQP_VHOST: "{{ cleanuri_amqp_vhost }}" + EXTRACTION_TASK_QUEUE: "{{ cleanuri_amqp_retrieval }}" + + +- name: Setup proxy site the CleanURI API Gateway + include_role: + name: setup-http-site-proxy + vars: + site_name: "{{ cleanuri_api_domain }}" + proxy_port: "{{ cleanuri_api_host_port }}" From 3d654427ac3930f45dd00bd822edb010d635daae Mon Sep 17 00:00:00 2001 From: Stefan Haun Date: Wed, 14 Sep 2022 22:26:43 +0200 Subject: [PATCH 2/5] :sparkles: Add host cleanuri-api to HTTPS forwarding --- holmium.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/holmium.yml b/holmium.yml index f8b5efe..bb59766 100644 --- a/holmium.yml +++ b/holmium.yml @@ -22,6 +22,7 @@ hosts: - name: gitea.n39.eu - name: uritools.n39.eu + - name: uritools-api.n39.eu - name: sl.n39.eu - name: pad.n39.eu - name: brotherql-web.n39.eu From f4544b255545416256823cd8f764eddf45027ecf Mon Sep 17 00:00:00 2001 From: Stefan Haun Date: Wed, 14 Sep 2022 22:46:47 +0200 Subject: [PATCH 3/5] :sparkles: Add external CleanURI setup to inventory --- inventory.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/inventory.yml b/inventory.yml index 67ebf70..2382932 100644 --- a/inventory.yml +++ b/inventory.yml @@ -68,6 +68,19 @@ all: 3461 pottwal.n39.eu: server_admin: "admin+pottwal@netz39.de" + + # These values need to be set up in RabbitMQ + cleanuri_amqp_host: "rabbitmq.n39.eu" + cleanuri_amqp_user: "cleanuri" + cleanuri_amqp_pass: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 36623438333666666263616562386561383564343534383130633336663130323562316337623532 + 6262336637646435666334653834643535316463366132320a393265616331376465383766643539 + 65656130356132333832396266313939326333323161316163653335376266303239663534303731 + 3666383966383463350a616461666534656232666235323333313139353363663763633261346533 + 64623232626531616235386235313763336465366466343334316361396239636337 + cleanuri_amqp_vhost: "/cleanuri" + shlink_geolite_license_key: !vault | $ANSIBLE_VAULT;1.1;AES256 33626234393039623132663736656363356562383235353737313034613630626339303263366665 From 69cce57024dae7227500331a9b9163b2467f86ff Mon Sep 17 00:00:00 2001 From: Stefan Haun Date: Wed, 14 Sep 2022 22:47:02 +0200 Subject: [PATCH 4/5] :sparkles: Switch to new CleanURI (uritools) implementation --- pottwal.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/pottwal.yml b/pottwal.yml index 8d72f2a..939f3a8 100644 --- a/pottwal.yml +++ b/pottwal.yml @@ -8,7 +8,7 @@ data_dir: "/srv/data" gitea_host_port: 9091 - uritools_host_port: 8080 + shlink_host_port: 8083 shlink_domain_name: sl.n39.eu @@ -32,6 +32,13 @@ - role: penguineer.dehydrated_cron - role: dd24-dyndns-cron # variables are set in the inventory + - role: cleanuri + vars: + cleanuri_ui_domain: uritools.n39.eu + cleanuri_ui_host_port: 8090 + cleanuri_api_domain: uritools-api.n39.eu + cleanuri_api_host_port: 8091 + # RabbitMQ setup can be found in the inventory tasks: @@ -89,25 +96,6 @@ - 3142:3142 - - name: Ensure container for URI tools is running - docker_container: - name: uritools - image: mrtux/clean_uri:0.4.1 - pull: true - state: started - detach: yes - ports: - - "127.0.0.1:{{ uritools_host_port }}:8080" - restart_policy: unless-stopped - - - name: Setup proxy site uritools.n39.eu - include_role: - name: setup-http-site-proxy - vars: - site_name: uritools.n39.eu - proxy_port: "{{ uritools_host_port }}" - - - name: Ensure container for shlink is running docker_container: name: shlink From d874aab674fee8b86df87380b97c6090af5734bd Mon Sep 17 00:00:00 2001 From: Stefan Haun Date: Wed, 14 Sep 2022 22:47:56 +0200 Subject: [PATCH 5/5] :sparkles: Add uritools-api to dehydrated --- pottwal.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/pottwal.yml b/pottwal.yml index 939f3a8..b5c796d 100644 --- a/pottwal.yml +++ b/pottwal.yml @@ -27,6 +27,7 @@ dehydrated_domains: - name: gitea.n39.eu - name: uritools.n39.eu + - name: uritools-api.n39.eu - name: sl.n39.eu - name: pad.n39.eu - role: penguineer.dehydrated_cron