CHANGELOG.md

@@ -12,6 +12,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 
+## [1.3.4] - 2019-10-14
+
+### Added
+### Changed
+
+- use kube-lego for TLS certs for clarin(-dev).eurac.edu
+- disable k8s-internal secure TLS communication: getting rid of the necessity
+  to (re-)create valid/self-signed certs
+  Note: using HTTP over port 443 for internal communication (Shibboleth mishap)
+- add IdP metadata from CLARIN server later in docker build build
+- Some changes to the liveness probe / "cronjobs" (see #60)
+
+### Removed
+### Fixed
+
+- #62
+
 ## [1.3.3] - 2019-07-23
 
 ### Changed
@@ -105,7 +122,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 This is the first release. It all starts here.
 
-[Unreleased]:https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3.3...HEAD
+[Unreleased]:https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3.4...HEAD
+[1.3.4]: https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3.3...v1.3.4 
 [1.3.3]: https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3.2...v1.3.3 
 [1.3.2]: https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3.1...v1.3.2 
 [1.3.1]: https://gitlab.inf.unibz.it/commul/docker/clarin-dspace/compare/v1.3...v1.3.1

DEPLOYMENT.md

@@ -7,7 +7,7 @@
 - [ ] (Optional) You can now deploy stable to dspace-dev (`./release.sh $VERSION_NUMBER-RC-$RCNUMBER`)
 - [ ] Make sure that the stable branch is clean and all commits are pushed to origin
 - [ ] `git checkout production`
-- [ ] `git checkout -b v$VERSION_NUMBER $HASHID -- .`
+- [ ] `git checkout -b $VERSION_NUMBER $HASHID`
       The $HASHID being the commit were the last merge from stable happened (before clarindev2clarin.sh changes were committed)
 - [ ] `git merge --no-ff stable`
 - [ ] resolve conflicts (though there shouldn't be any) and commit
@@ -15,7 +15,7 @@
 - [ ] Run the clarindev2clarin.sh script
 - [ ] `git commit -a `
 - [ ] `git checkout production`
-- [ ] `git merge v1.3-$VERSION_NUMBER`
+- [ ] `git merge $VERSION_NUMBER`
 - [ ] resolve conflicts and commit
 - [ ] `git tag -a $VERSION_NUMBER -s (-u $GPG_ID_HASH)` defaults to your default GPG key
 - [ ] `git branch -d $VERSION_NUMBER`

clarin-dev2clarin.sh

@@ -3,7 +3,7 @@
 # outside the docker
 perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/shibboleth2.xml
 perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/local.properties
-perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/default-ssl
+perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/nginx.default.conf
 
 perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/aai.js
 perl -pi -e 's/clarin-dev/clarin/g' dockerfiles/commul-customization/aai_config.js

dockerfiles/Dockerfile.nginx

@@ -76,8 +76,6 @@ RUN /tmp/shibboleth_sp_with_fastcgi.sh
 # copy the init script
 RUN cp /opt/shibboleth-sp-fastcgi/etc/shibboleth/shibd-debian /etc/init.d/shibd
 RUN chmod a+x /etc/init.d/shibd
-# add IdP metadata from CLARIN server
-ADD https://infra.clarin.eu/aai/prod_md_about_spf_idps.xml /opt/shibboleth-sp-fastcgi/var/cache/shibboleth/ 
 
 # create the test secure folder and set up perl fastcgi
 RUN mkdir /opt/nginx/html/secure
@@ -101,7 +99,7 @@ COPY commul-customization/php-fpm.conf /etc/php/7.0/fpm/
 COPY commul-customization/php.ini /etc/php/7.0/fpm/
 
 # copy over config files
-COPY commul-customization/default-ssl /opt/nginx/conf/
+COPY commul-customization/nginx.default.conf /opt/nginx/conf/
 COPY commul-customization/nginx.conf /opt/nginx/conf/
 COPY commul-customization/repository_auth /opt/nginx/conf/
 COPY commul-customization/shibboleth2.xml /opt/shibboleth-sp-fastcgi/etc/shibboleth/
@@ -126,5 +124,7 @@ RUN yui-compressor -o aai.min.js aai.js
 # COPY commul-customization/certs/clarin.key /etc/ssl/private/
 # COPY commul-customization/certs/clarin_eurac_edu.crt /etc/ssl/certs/
 
+# add IdP metadata from CLARIN server
+ADD https://infra.clarin.eu/aai/prod_md_about_spf_idps.xml /opt/shibboleth-sp-fastcgi/var/cache/shibboleth/
 
 ENTRYPOINT ["/usr/local/bin/supervisord", "-c", "/etc/supervisord.conf"]

dockerfiles/commul-customization/local.properties

@@ -128,7 +128,7 @@ mail.server.disabled = false
 mail.server.username = ${MAIL_USER}
 mail.server.password = ${MAIL_PASSWORD}
 mail.server.port = 587
-mail.extraproperties = mail.smtp.starttls.enable=true
+mail.extraproperties = mail.smtp.starttls.enable=true,mail.smtp.ssl.protocols=TLSv1.2
 
 
 # Eurac IAL: Contact information

dockerfiles/commul-customization/nginx.conf

@@ -65,5 +65,5 @@ http {
         text/xml
         text/x-component;
 
-    include	 default-ssl;
+    include	 nginx.default.conf;
 }

dockerfiles/commul-customization/default-ssl → dockerfiles/commul-customization/nginx.default.conf

@@ -4,9 +4,11 @@ upstream tomcats {
 }
 
 server {
-  listen 80;
-  listen [::]:80;
-  server_name clarin.eurac.edu;
+  listen 443;
+  listen [::]:443;
+  server_name clarin.eurac.edu localhost;
+
+  root  /opt/nginx/html;
 
   location /repository/oai {
 
@@ -22,29 +24,6 @@ server {
 
   }
 
-  location / {
-    return 301 https://$server_name$request_uri;
-  }
-}
-
-server {
-  listen 443;
-  server_name clarin.eurac.edu localhost;
-
-  root  /opt/nginx/html;
-  index index.html index.htm;
-
-  ssl on;
-  ssl_certificate /etc/ssl/clarin/clarin_eurac_edu.crt;
-  ssl_certificate_key /etc/ssl/clarin/clarin.key;
-
-  ssl_protocols        TLSv1 TLSv1.1 TLSv1.2;
-  ssl_ciphers RC4:HIGH:!aNULL:!MD5;
-  ssl_prefer_server_ciphers on;
-  keepalive_timeout    70;
-  ssl_session_cache    shared:SSL:10m;
-  ssl_session_timeout  10m;
-
   add_header 'Access-Control-Allow-Origin' "*";
   add_header 'Access-Control-Allow-Credentials' 'true';
 
@@ -81,10 +60,6 @@ server {
 
   location / {
     index index.html index.htm;
-
-    if ($HTTP_X_SCHEME = "http") {
-        rewrite ^ https://$http_host$request_uri permanent;
-    }
   }
 
   location /php/aa-statistics.php {

dockerfiles/commul-customization/repository_auth

@@ -19,9 +19,11 @@ more_clear_input_headers
 more_clear_input_headers 'affiliation' 'assurance' 'cn' 'eduPersonOrgUnitDN' 'eduPersonStudyiProgramme' 'eduPersonStudySubject' 'entitlement' 'eppn' 'givenName' 'mail' 'o' 'ou' 'persistent-id' 'sn' 'telephoneNumber' 'unscoped-affiliation';
 
 # Require https and will redirect
-if ($https != "on") {
-  return 301 https://$http_host$request_uri;
-}
+#if ($https != "on") {
+#  return 301 https://$http_host$request_uri;
+#}
+# FIXME: reverse proxy-ing -> check connection params to revproxy and enforce
+# http->https i if *there* is no https.
 
 shib_request /shibauthorizer;
 shib_request_use_headers on;

dockerfiles/commul-customization/robots-clarin.txt

@@ -16,6 +16,8 @@ Disallow: /repository/xmlui/forgot
 Disallow: /repository/xmlui/login
 Disallow: /repository/xmlui/register
 Disallow: /repository/xmlui/search-filter
+Disallow: /repository/*allzip$
+Disallow: /repository/oai/requeststripped
 Disallow: /Shibboleth.sso
 #
 # Optionally uncomment the following line ONLY if sitemaps are working

dockerfiles/commul-customization/shibboleth2.xml

@@ -24,10 +24,10 @@
     <RequestMapper type="XML">
     <RequestMap>
         <Host name="clarin.eurac.edu"
-                authType="shibboleth"
-                requireSession="true"
-		exportAssertion="true"
-                redirectToSSL="443">
+            authType="shibboleth"
+            requireSession="true"
+            exportAssertion="true"
+            redirectToSSL="443">
             <Path name="/secure" />
         </Host>
 
@@ -36,9 +36,9 @@
 
     <!-- The ApplicationDefaults element is where most of Shibboleth's SAML bits are defined. -->
     <ApplicationDefaults entityID="https://clarin.eurac.edu/Shibboleth.sso/Metadata"
-                         REMOTE_USER="eppn persistent-id targeted-id"
-			 sessionHook="/php/aa-statistics.php"
-                         cipherSuites="ECDHE+AESGCM:ECDHE:!aNULL:!eNULL:!LOW:!EXPORT:!RC4:!SHA:!SSLv2">
+        REMOTE_USER="eppn persistent-id targeted-id"
+        sessionHook="/php/aa-statistics.php"
+        cipherSuites="ECDHE+AESGCM:ECDHE:!aNULL:!eNULL:!LOW:!EXPORT:!RC4:!SHA:!SSLv2">
 
         <!--
         Controls session lifetimes, address checks, cookie handling, and the protocol handlers.
@@ -50,13 +50,13 @@
         security of your site. Stealing sessions via cookie theft is much easier with this disabled.
         -->
         <Sessions lifetime="28800"
-		  timeout="3600"
-		  relayState="ss:mem"
-                  checkAddress="false"
-                  handlerSSL="true"
-		  cookieProps="https"
-		  exportLocation="/GetAssertion"
-                  exportACL="127.0.0.1">
+            timeout="3600"
+            relayState="ss:mem"
+            checkAddress="false"
+            handlerSSL="false"
+            cookieProps="https"
+            exportLocation="/GetAssertion"
+            exportACL="127.0.0.1">
 
             <!--
             Configures SSO for a default IdP. To allow for >1 IdP, remove
@@ -65,8 +65,8 @@
             You can also override entityID on /Login query string, or in RequestMap/htaccess.
             -->
             <SSO
-		discoveryProtocol="SAMLDS" discoveryURL="https://discovery.clarin.eu/discojuice">
-              SAML2 SAML1
+                discoveryProtocol="SAMLDS" discoveryURL="https://discovery.clarin.eu/discojuice">
+                SAML2 SAML1
             </SSO>
 
 <!--            <SSO entityID="https://idp.eurac.edu/idp/shibboleth">

kubernetes/dspace-deployment.yaml

@@ -18,6 +18,12 @@ spec:
       labels:
         io.kompose.service: dspace
     spec:
+      initContainers:
+      - image: postgres
+        name: clarin-dspace-check-db-ready
+        command: ['sh', '-c',
+          'until pg_isready -h postgres -p 5432;
+          do echo waiting for database; sleep 2; done;']
       containers:
       - env:
         - name: DSPACE_USER
@@ -50,7 +56,7 @@ spec:
             secretKeyRef:
               key: mail.pass
               name: dspace-secrets
-        image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/dspace:1.3.3
+        image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/dspace:1.3.4
         name: clarin-dspace
         imagePullPolicy: Always
         ports:
@@ -63,12 +69,12 @@ spec:
         - mountPath: /opt/lindat-dspace/installation/assetstore
           name: dspace-assetstore
         livenessProbe:
-          httpGet:
-            host: clarin.eurac.edu
-            path: /repository/xmlui/
-            port: 443
-            scheme: HTTPS
-          initialDelaySeconds: 600
+          exec:
+            command:
+              - /bin/sh
+              - -c
+              - reply=$(curl -s -o /dev/null -w %{http_code} https://clarin.eurac.edu/repository/xmlui/); if [ "$reply" -lt 200 -o "$reply" -ge 400 ]; then exit 1; fi; ps aux | grep -v grep | grep -E "sleep|timeout" && exit 0 || exit 1;
+          initialDelaySeconds: 900
           periodSeconds: 15
           timeoutSeconds: 3
       restartPolicy: Always

kubernetes/dspace-ingress.yaml

@@ -17,7 +17,7 @@ metadata:
       if ($rewrite_https = 1) {
         return 301 https://$server_name$request_uri;
       }
-    ingress.kubernetes.io/secure-backends: "true"
+    kubernetes.io/tls-acme: "true"
   name: dspace-ingress
   namespace: dspace
 spec:
@@ -26,10 +26,19 @@ spec:
     http:
       paths:
       - backend:
+        # we are deliberately misusing port:443 *without* SSL, i.e. *NO*
+        # ingress.kubernetes.io/secure-backends: "true"
+        # This will result in https:// redirectes generated from shibboleth
+        # later on (without the need to change http:// to https://)
+        # see https://github.com/nginx-shib/nginx-http-shibboleth/issues/22
+        #
+        # For a general overview for SP reverse proxy-ing
+        # see https://wiki.shibboleth.net/confluence/display/SHIB2/SPReverseProxy
+        #
           serviceName: nginx
           servicePort: 443
         path: /
   tls:
   - hosts:
     - clarin.eurac.edu
-    secretName: host-tls-cert
+    secretName: clarin-eurac-edu-tls

kubernetes/nginx-deployment.yaml

@@ -20,18 +20,18 @@ spec:
         io.kompose.service: nginx
     spec:
       containers:
-      - image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/nginx:1.3.3
+      - image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/nginx:1.3.4
         imagePullPolicy: Always
         name: clarin-nginx
         ports:
-        - containerPort: 80
         - containerPort: 443
         resources: {}
         volumeMounts:
         - mountPath: /opt/shibboleth-sp-fastcgi/etc/shibboleth/certs
           name: nginx-shib-certs
-        - mountPath: /etc/ssl/clarin
+        - mountPath: /etc/ssl/nginx
           name: nginx-ssl-certs
+          readOnly: true
         livenessProbe:
           httpGet:
             host: clarin.eurac.edu
@@ -44,18 +44,9 @@ spec:
       imagePullSecrets:
         - name: gitlab-scientificnet-org-registry
       volumes:
-        - cephfs:
-            monitors:
-            - 10.8.55.201:6789
-            - 10.8.55.202:6789
-            - 10.8.55.203:6789
-            - 10.7.55.201:6789
-            - 10.7.55.202:6789
-            path: /eurac/commul/projects/clarin/kubernetes/dspace/ssl-certs
-            secretRef:
-              name: client.fs.commul.admin
-            user: fs.commul.admin
-          name: nginx-ssl-certs
+        - name: nginx-ssl-certs
+          secret:
+            secretName: clarin-eurac-edu-tls
         - cephfs:
             monitors:
             - 10.8.55.201:6789

kubernetes/nginx-service.yaml

@@ -11,9 +11,6 @@ metadata:
   namespace: dspace
 spec:
   ports:
-  - name: "80"
-    port: 80
-    targetPort: 80
   - name: "443"
     port: 443
     targetPort: 443

kubernetes/postgres-deployment.yaml

@@ -48,7 +48,7 @@ spec:
             secretKeyRef:
               key: dspace.utils.name
               name: dspace-secrets
-        image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/postgres:1.3.3
+        image: gitlab.inf.unibz.it:4567/commul/docker/clarin-dspace/production/postgres:1.3.4
         imagePullPolicy: Always
         name: clarin-postgres
         resources: {}