Initial commit

docker-compose.yml

+version: '3.4'
+services:
+
+  zookeeper:
+    image: confluentinc/cp-zookeeper:7.0.1
+    container_name: sn-zookeeper
+    ports:
+      - '${PORT_ZOOKEEPER:-2181}:2181'
+    volumes:
+      - zookeeper-data:/var/lib/zookeeper/data
+      - zookeeper-logs:/var/lib/zookeeper/log
+      - zookeeper-secrets:/etc/zookeeper/secrets
+    environment:
+      ZOOKEEPER_SERVER_ID: '1'
+      ZOOKEEPER_CLIENT_PORT: '2181'
+      ZOOKEEPER_TICK_TIME: '2000'
+
+  broker1:
+    image: confluentinc/cp-kafka:7.0.1
+    container_name: sn-broker1
+    depends_on:
+      - zookeeper
+    ports:
+      - '${PORT_BROKER1:-29092}:29092'
+    volumes:
+      - broker1-data:/var/lib/kafka/data
+    environment:
+      KAFKA_BROKER_ID: '1'
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENERS: 'INTERNAL://broker1:9092,EXTERNAL://broker1:29092'
+      KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://broker1:9092,EXTERNAL://localhost:${PORT_BROKER1:-29092}'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'INTERNAL'
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
+      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: '1'
+      KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN'
+      KAFKA_LOG4J_LOGGERS: "\
+        kafka=WARN,\
+        kafka.controller=WARN,\
+        kafka.log.LogCleaner=WARN,\
+        state.change.logger=WARN,\
+        kafka.producer.async.DefaultEventHandler=WARN"
+
+
+  connect1:
+    image: confluentinc/cp-kafka-connect:7.0.1
+    container_name: sn-connect1
+    depends_on:
+      - broker1
+    ports:
+      - '${PORT_CONNECT:-8083}:8083'
+    volumes:
+      - connect1-plugins:/usr/share/confluent-hub-components
+      - ./kafka-connect/kafka-connect-entrypoint.sh:/entrypoint.sh
+    entrypoint: /entrypoint.sh
+    environment:
+      # Network configuration
+      CONNECT_BOOTSTRAP_SERVERS: 'broker1:9092'
+      CONNECT_REST_PORT: '8083'
+      CONNECT_REST_ADVERTISED_HOST_NAME: 'connect1'
+      # Location where plugins are stored (we map it via a volume to a host directory, so place connectors there)
+      CONNECT_PLUGIN_PATH: '/usr/share/java,/usr/share/confluent-hub-components'
+      # Connect workers belong to groups, and each group store its internal data in some Kafka topics configured here
+      CONNECT_GROUP_ID: 'connectgroup1'
+      CONNECT_CONFIG_STORAGE_TOPIC: '_connectgroup1_configs'
+      CONNECT_OFFSET_STORAGE_TOPIC: '_connectgroup1_offset'
+      CONNECT_STATUS_STORAGE_TOPIC: '_connectgroup1_status'
+      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1'
+      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1'
+      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1'
+      # Logging configuration
+      CONNECT_LOG4J_ROOT_LOGLEVEL: 'INFO'
+      # Converter configuration for connector data - choose one of the following
+      # -- (1) StringConverter --
+      # CONNECT_KEY_CONVERTER: 'org.apache.kafka.connect.storage.StringConverter'
+      # CONNECT_VALUE_CONVERTER: 'org.apache.kafka.connect.storage.StringConverter'
+      # -- (2) JsonConverter --
+      CONNECT_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
+      CONNECT_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter'
+
+  ws-proxy:
+    image: kpmeen/kafka-websocket-proxy:1.1.1
+    container_name: sn-ws-proxy
+    depends_on:
+      - broker1
+    ports:
+      - '${PORT_WS_PROXY:-28078}:8078'
+    environment:
+      WSPROXY_PORT: '8078'
+      WSPROXY_KAFKA_BOOTSTRAP_HOSTS: 'broker1:9092'
+      WSPROXY_SESSION_STATE_TOPIC: '_wsproxy.session.state'
+      WSPROXY_SESSION_STATE_REPLICATION_FACTOR: '3'
+      # more info at: https://kpmeen.gitlab.io/kafka-websocket-proxy/
+
+  ws-proxy2:
+    image: kpmeen/kafka-websocket-proxy:1.1.1
+    container_name: sn-ws-proxy2
+    depends_on:
+      - broker1
+    ports:
+      - '${PORT_WS_PROXY:-28079}:8079'
+    environment:
+      WSPROXY_PORT: '8079'
+      WSPROXY_KAFKA_BOOTSTRAP_HOSTS: 'broker1:9092'
+      WSPROXY_SESSION_STATE_TOPIC: '_wsproxy.session.state'
+      WSPROXY_SESSION_STATE_REPLICATION_FACTOR: '3'
+      # more info at: https://kpmeen.gitlab.io/kafka-websocket-proxy/
+
+  kafka-ui:
+    image: provectuslabs/kafka-ui:0.3.3
+    container_name: sn-kafka-ui
+    ports:
+      - '${PORT_KAFKA_UI:-28080}:8080'
+    depends_on:
+      - zookeeper
+      - broker1
+    environment:
+      KAFKA_CLUSTERS_0_NAME: 'cluster1'
+      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: 'broker1:9092'
+      KAFKA_CLUSTERS_0_ZOOKEEPER: 'zookeeper:2181'
+      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: connect1
+      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://connect1:8083
+      LOGGING_LEVEL_ROOT: 'INFO'
+      LOGGING_LEVEL_COM_PROVECTUS: 'INFO'
+
+  flink-jobmanager:
+    image: ${FLINK_IMAGE:-flink:1.14.4-scala_2.12}
+    container_name: flink-jobmanager
+    ports:
+      - '${PORT_FLINK:-8081}:8081'
+    volumes:
+      - .:/data
+    command: jobmanager
+    environment:
+      FLINK_PROPERTIES: |
+        jobmanager.rpc.address: flink-jobmanager
+
+  flink-taskmanager1:
+    image: ${FLINK_IMAGE:-flink:1.14.4-scala_2.12}
+    container_name: flink-taskmanager1
+    command: taskmanager
+    environment:
+      FLINK_PROPERTIES: |
+        jobmanager.rpc.address: flink-jobmanager
+        taskmanager.numberOfTaskSlots: ${FLINK_TASK_SLOTS:-4}
+        parallelism.default: ${FLINK_DEFAULT_PARALLELISM:-1}
+
+volumes:
+  zookeeper-data:
+  zookeeper-logs:
+  zookeeper-secrets:
+  broker1-data:
+  connect1-plugins:

flink-job/.idea/.gitignore

+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

flink-job/.idea/compiler.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <annotationProcessing>
+      <profile name="Maven default annotation processors profile" enabled="true">
+        <sourceOutputDir name="target/generated-sources/annotations" />
+        <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
+        <outputRelativeToContentRoot value="true" />
+        <module name="flink-job" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>
\ No newline at end of file

flink-job/.idea/encodings.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Encoding">
+    <file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
+    <file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
+  </component>
+</project>
\ No newline at end of file

flink-job/.idea/jarRepositories.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RemoteRepositoriesConfiguration">
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Central Repository" />
+      <option name="url" value="https://repo.maven.apache.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="central" />
+      <option name="name" value="Maven Central repository" />
+      <option name="url" value="https://repo1.maven.org/maven2" />
+    </remote-repository>
+    <remote-repository>
+      <option name="id" value="jboss.community" />
+      <option name="name" value="JBoss Community repository" />
+      <option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
+    </remote-repository>
+  </component>
+</project>
\ No newline at end of file

flink-job/.idea/misc.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ExternalStorageConfigurationManager" enabled="true" />
+  <component name="MavenProjectsManager">
+    <option name="originalFiles">
+      <list>
+        <option value="$PROJECT_DIR$/pom.xml" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_11" default="true" project-jdk-name="11" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>
\ No newline at end of file

flink-job/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>flink-job</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <flink.version>1.14.4</flink.version>
+        <scala.binary.version>2.11</scala.binary.version>
+        <log4j.version>2.17.1</log4j.version>
+        <slf4j.version>1.7.25</slf4j.version>
+    </properties>
+
+    <dependencies>
+        <!-- FLINK DEPENDENCIES -->
+
+        <!-- add the following dependencies always, to use Flink -->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-java</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-clients_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- add the following to optionally use the Flink Table API -->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-api-java</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- add the following to optionally use Flink CEP -->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-cep_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+
+        <!-- add the following to optionally use the Flink connector for Kafka -->
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
+            <version>${flink.version}</version>
+        </dependency>
+
+        <!-- OTHER DEPENDENCIES -->
+
+        <!-- Logging in IDE via log4j2 (libraries excluded from generaged JAR too) -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.13.2.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${slf4j.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j-impl</artifactId>
+            <version>${log4j.version}</version>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-api</artifactId>
+            <version>${log4j.version}</version>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>${log4j.version}</version>
+            <scope>runtime</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.1.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <excludes>
+                                    <exclude>org.apache.flink:flink-shaded-force-shading</exclude>
+                                    <exclude>com.google.code.findbugs:jsr305</exclude>
+                                    <exclude>org.slf4j:*</exclude>
+                                    <exclude>org.apache.logging.log4j:*</exclude>
+                                </excludes>
+                            </artifactSet>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>it.unibz.inf.rtbdp.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file

flink-job/target/maven-archiver/pom.properties

+#Generated by Maven
+#Mon Jun 13 13:49:14 CEST 2022
+groupId=org.example
+artifactId=flink-job
+version=1.0-SNAPSHOT

flink-job/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst

+FlinkManager$1.class