cleanup

config/mastodon-sink-s3.json

@@ -1,16 +0,0 @@
-{
-    "name": "mastodon-sink-s3",
-    "connector.class": "io.confluent.connect.s3.S3SinkConnector",
-    "topics": "mastodon-topic",
-    "format.class": "io.confluent.connect.s3.format.parquet.ParquetFormat",
-    "flush.size": "1000",
-    "s3.bucket.name": "mastodon",
-    "aws.access.key.id": "minio",
-    "aws.secret.access.key": "minio123",
-    "storage.class": "io.confluent.connect.s3.storage.S3Storage",
-    "store.url": "http://localhost:9000"
-}
-
-  
-
-

kafkaproducer.py

@@ -22,28 +22,12 @@ def kafka_producer():
     producer = AvroProducer(producer_config, default_value_schema=value_schema)
     return 'mastodon-topic', producer
 
-    # try:
-        
-
-    #     value_dict = {  'language': 'en', 'favourites': 0, 'username': 'bob', 'bot': False, 'tags': 0, 'characters': 50, 'words': 12}
-
-    #     producer.produce(topic = 'mastodon-topic', value = value_dict)
-    #     producer.flush()
-
-    # except KafkaException as e:
-    #     print('Kafka failure ' + e)
-
-
-
 def main():
+    # example test producer
     topic_name, producer = kafka_producer()
 
     value_dict = {  'language': 'en', 'favourites': 0, 'username': 'bob', 'bot': False, 'tags': 0, 'characters': 50, 'words': 12}
     producer.produce(topic = topic_name, value = value_dict)
-
-    value_dict = {  'language': 'fr', 'favourites': 0, 'username': 'jane', 'bot': False, 'tags': 0, 'characters': 500, 'words': 120}
-    producer.produce(topic = topic_name, value = value_dict)
-
     producer.flush()
 
 if __name__ == '__main__':

shell/go.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-BASE=${HOME}/git/saubury/mastodon-stream/
-PY=./env/bin/python
-cd ${BASE}
-
- 
-
-# while true; do echo Start; ${PY} mastodonlisten.py --enableKafka --public; sleep 30; done &
-
-while true; do echo Start; ${PY} mastodonlisten.py --baseURL https://mastodon.social       --enableKafka --watchdog 30 --public; sleep 30; done
-while true; do echo Start; ${PY} mastodonlisten.py --baseURL https://hachyderm.io          --enableKafka --watchdog 600 ; sleep 30; done
-while true; do echo Start; ${PY} mastodonlisten.py --baseURL https://mastodon.au/          --enableKafka --watchdog 600 ; sleep 30; done
-while true; do echo Start; ${PY} mastodonlisten.py --baseURL https://data-folks.masto.host --enableKafka --watchdog 600 ; sleep 30; done
-