Add initial poc

This code was started in 2019, updated in 2022 and expanded with tracing in 2023/2024.

In principal it works and is compatible with the production ClueBot NG bot (https://github.com/cluebotng/bot), in reality it needs a lot of cleanup.

.github/config.yaml

@@ -0,0 +1,37 @@
+bot:
+  owner: Cobi
+  friends:
+    - ClueBot
+    - DASHBotAV
+  run: false
+  angry: true
+
+wikipedia:
+  host: en.wikipedia.org
+  username: ClueBot_NG
+  owner: NaomiAmethyst
+
+irc:
+  server: irc.freenode.org
+  port: 6697
+  username: CBNGRelay-CI
+  channel:
+    spam: wikipedia-en-cbngfeed-ci
+    revert: wikipedia-en-cbngrevertfeed-ci
+    debug: wikipedia-en-cbngdebug-ci
+
+sql:
+  replica:
+    host: 127.0.0.1
+    port: 3306
+    schema: enwiki_p
+    username: root
+  cluebot:
+    host: 127.0.0.1
+    port: 3306
+    schema: cbng
+    username: root
+
+core:
+  host: 127.0.0.1
+  port: 3565

.github/workflows/ci.yml

@@ -1,17 +1,17 @@
 name: Continuous Integration
 on:
-  push: {branches: [main]}
-  pull_request: {}
+  push: { branches: [ main ] }
+  pull_request: { }
 permissions:
   contents: read
 jobs:
   build:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.16'
+          go-version: '^1.23'
       - run: go build
   test:
     runs-on: ubuntu-20.04
@@ -24,10 +24,10 @@ jobs:
         ports:
           - '3306:3306'
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.16'
+          go-version: '^1.23'
       - name: Setup MySQL
         run: |
           while true;
@@ -49,16 +49,16 @@ jobs:
   vet:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.16'
+          go-version: '^1.23'
       - run: go vet
   golangci:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.16'
-      - uses: golangci/golangci-lint-action@v2
+          go-version: '^1.23'
+      - uses: golangci/golangci-lint-action@v6

.github/workflows/release.yml

@@ -1,17 +1,17 @@
 name: Release
 on:
-  push: {tags: ['*']}
+  push: { tags: [ '*' ] }
 permissions:
   deployments: write
   contents: write
 jobs:
   release:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.16'
+          go-version: '^1.23'
       - run: |
           go build \
           -ldflags="-X 'github.com/cluebotng/botng/pkg/cbng/config.ReleaseTag=${{ github.ref }}'"

README.md

@@ -5,6 +5,7 @@ Proof of concept rewrite of ClueBot NG's bot component (https://github.com/clueb
 
 Goals
 -----
+
 * Maintain revert outcome; specifically the pre-filtering that happens before core scoring
 * Improve resource usage; specifically MySQL connection pooling, ideally removing the pre-fork model
 * Improve debugging; expose metrics for health, which currently are calculated from scraping logs
@@ -15,6 +16,17 @@ Compatibility
 -------------
 
 Not supported:
-* CBAutostalk.js
-* CBAutoedit.js
-* oftenvandalized.txt
+
+* `CBAutostalk.js` & `CBAutoedit.js` - These were used to emit messages into IRC, but haven't been used since 2018 (
+  80cab4)
+* `oftenvandalized.txt` - This was used to emit messages into IRC, but hasn't been used since, but haven't been used
+  since 2018 (80cab4)
+* `irc.wikimedia.org` - In favour of the HTTP event stream (which backs the IRC relay)
+
+TODO
+----
+
+* Add test cases for each section of functionality
+* Run side by side in production & compare decisions
+* Cleanup metric production
+* Cleanup logging/tracing code

cluebot.sql

@@ -17,55 +17,59 @@
 -- Table structure for table `beaten`
 --
 DROP TABLE IF EXISTS `beaten`;
-CREATE TABLE `beaten` (
-  `id` int(11) NOT NULL auto_increment,
-  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
-  `article` varchar(256) NOT NULL,
-  `diff` varchar(512) NOT NULL,
-  `user` varchar(256) NOT NULL,
-  PRIMARY KEY  (`id`)
+CREATE TABLE `beaten`
+(
+    `id`        int(11) NOT NULL auto_increment,
+    `timestamp` timestamp    NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+    `article`   varchar(256) NOT NULL,
+    `diff`      varchar(512) NOT NULL,
+    `user`      varchar(256) NOT NULL,
+    PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 --
 -- Table structure for table `trr`
 --
 DROP TABLE IF EXISTS `trr`;
-CREATE TABLE `trr` (
-  `id` int(11) NOT NULL auto_increment,
-  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
-  `user` varchar(256) NOT NULL,
-  `title` varchar(256) NOT NULL,
-  `url` varchar(512) NOT NULL,
-  `revid` int(11) NOT NULL,
-  `md5` char(32) default NULL,
-  PRIMARY KEY  (`id`)
+CREATE TABLE `trr`
+(
+    `id`        int(11) NOT NULL auto_increment,
+    `timestamp` timestamp    NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+    `user`      varchar(256) NOT NULL,
+    `title`     varchar(256) NOT NULL,
+    `url`       varchar(512) NOT NULL,
+    `revid`     int(11) NOT NULL,
+    `md5`       char(32)              default NULL,
+    PRIMARY KEY (`id`)
 ) ENGINE=MyISAM AUTO_INCREMENT=488749 DEFAULT CHARSET=latin1;
 --
 -- Table structure for table `vandalism`
 --
 DROP TABLE IF EXISTS `vandalism`;
-CREATE TABLE `vandalism` (
-  `id` int(11) NOT NULL auto_increment,
-  `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
-  `user` varchar(256) NOT NULL,
-  `article` varchar(256) NOT NULL,
-  `heuristic` varchar(64) NOT NULL,
-  `regex` varchar(2048) default NULL,
-  `reason` varchar(512) NOT NULL,
-  `diff` varchar(512) NOT NULL,
-  `old_id` int(11) NOT NULL,
-  `new_id` int(11) NOT NULL,
-  `reverted` tinyint(1) NOT NULL,
-  PRIMARY KEY  (`id`)
+CREATE TABLE `vandalism`
+(
+    `id`        int(11) NOT NULL auto_increment,
+    `timestamp` timestamp    NOT NULL default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
+    `user`      varchar(256) NOT NULL,
+    `article`   varchar(256) NOT NULL,
+    `heuristic` varchar(64)  NOT NULL,
+    `regex`     varchar(2048)         default NULL,
+    `reason`    varchar(512) NOT NULL,
+    `diff`      varchar(512) NOT NULL,
+    `old_id`    int(11) NOT NULL,
+    `new_id`    int(11) NOT NULL,
+    `reverted`  tinyint(1) NOT NULL,
+    PRIMARY KEY (`id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
 --
 -- Table structure for table `cluster_node`
 --
 DROP TABLE IF EXISTS `cluster_node`;
-CREATE TABLE `cluster_node` (
-  `node` varchar(256) NOT NULL,
-  `port` int(11) NOT NULL,
-  `type` varchar(256) NOT NULL,
-  PRIMARY KEY  (`type`)
+CREATE TABLE `cluster_node`
+(
+    `node` varchar(256) NOT NULL,
+    `port` int(11) NOT NULL,
+    `type` varchar(256) NOT NULL,
+    PRIMARY KEY (`type`)
 ) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=latin1;
 /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
 /*!40101 SET SQL_MODE=@OLD_SQL_MODE */;

go.mod

@@ -1,44 +1,90 @@
 module github.com/cluebotng/botng
 
-go 1.18
+go 1.22.0
+
+toolchain go1.22.3
 
 require (
-	github.com/go-sql-driver/mysql v1.4.1
-	github.com/honeycombio/libhoney-go v1.10.0
-	github.com/prometheus/client_golang v1.0.0
+	github.com/go-sql-driver/mysql v1.8.1
+	github.com/prometheus/client_golang v1.20.5
 	github.com/satori/go.uuid v1.2.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.4.0
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20170531160350-a96e63847dc3
+	github.com/sirupsen/logrus v1.9.3
+	github.com/spf13/pflag v1.0.5
+	github.com/spf13/viper v1.19.0
+	golang.org/x/time v0.8.0
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
 require (
-	github.com/beorn7/perks v1.0.0 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
-	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
 	github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect
 	github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 // indirect
-	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fsnotify/fsnotify v1.4.7 // indirect
-	github.com/golang/protobuf v1.3.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.8.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/magiconair/properties v1.8.1 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/mitchellh/mapstructure v1.1.2 // indirect
-	github.com/pelletier/go-toml v1.4.0 // indirect
-	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
-	github.com/prometheus/common v0.6.0 // indirect
-	github.com/prometheus/procfs v0.0.2 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
-	github.com/spf13/cast v1.3.0 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
-	golang.org/x/text v0.3.2 // indirect
-	google.golang.org/appengine v1.6.1 // indirect
+	github.com/honeycombio/otel-config-go v1.17.0 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/lufia/plan9stats v0.0.0-20240513124658-fba389f38bae // indirect
+	github.com/magiconair/properties v1.8.9 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.61.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/sagikazarmark/locafero v0.6.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sethvargo/go-envconfig v1.1.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.6 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.7.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.14 // indirect
+	github.com/tklauser/numcpus v0.8.0 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
+	go.opentelemetry.io/contrib/instrumentation/host v0.53.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.53.0 // indirect
+	go.opentelemetry.io/contrib/propagators/b3 v1.28.0 // indirect
+	go.opentelemetry.io/contrib/propagators/ot v1.28.0 // indirect
+	go.opentelemetry.io/otel v1.32.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.28.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.8.0 // indirect
+	go.opentelemetry.io/otel/log v0.8.0 // indirect
+	go.opentelemetry.io/otel/metric v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.32.0 // indirect
+	go.opentelemetry.io/otel/sdk/log v0.8.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect
+	go.opentelemetry.io/otel/trace v1.32.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	golang.org/x/exp v0.0.0-20241204233417-43b7b7cde48d // indirect
+	golang.org/x/net v0.32.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
+	google.golang.org/grpc v1.65.0 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 	gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )