commit 111bb88d18a16b243d91f48e7143fde0769cbf34 Author: Martin Donnelly Date: Mon Sep 2 14:08:28 2024 +0100 Init diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ee78565 --- /dev/null +++ b/.gitignore @@ -0,0 +1,106 @@ +### Go template +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +go-blocklist + +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +### GoLand template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +/downloads/ +/public/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b6e5d16 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +PROJECT = go-blocklist + +VERSION="05" +BUILD=`date +%yw%Vb` + +ECR_REPO = git.caliban.io/martin + +APP_IMAGE = $(ECR_REPO)/$(PROJECT):$(BUILD)$(VERSION) + +NO_CACHE = true + +.PHONY: build +build: + + GCO_ENABLED=0 GOOS=linux go build -o ${PROJECT} main.go + +# docker build ./docker/. -t $(APP_IMAGE) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE) --compress=true + docker build --platform linux/amd64 --no-cache -force-rm --tag ${APP_IMAGE} --file ./docker/Dockerfile . + + +#push docker image to registry +.PHONY: push +push: build + docker push $(APP_IMAGE) + + +#push docker image to registry +.PHONY: run +run: build + docker run $(APP_IMAGE) + +ver: + @echo '$(VERSION)' + #echo $ERSION +.PHONY: ver \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d1d0ae9 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3.5' + +services: + blocklist: + container_name: blocklist + image: git.caliban.io/martin/go-blocklist:24w36b05 + ports: + - '8989:8989' + restart: always + volumes: + - ./public:/app/public + - ./downloads:/app/downloads + diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..408ad97 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,20 @@ +# FROM debian:12-slim +FROM alpine:latest + +WORKDIR /app + +RUN mkdir -p /app/downloads /app/public + +COPY ./go-blocklist /app/ + + + +# Need the following to get a go app to run inside a docker container +# as per: https://www.fairlyusefulcode.co.uk/post/go-alpine-linux/ +RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2 +RUN chmod +x /app/go-blocklist + +EXPOSE 8989 + +CMD [ "/app/go-blocklist"] + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..48619ba --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module go-blocklist + +go 1.22 + +require ( + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/gofiber/fiber/v2 v2.52.5 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/otiai10/copy v1.14.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/robfig/cron/v3 v3.0.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.15.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4ccc133 --- /dev/null +++ b/go.sum @@ -0,0 +1,33 @@ +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo= +github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= diff --git a/main.go b/main.go new file mode 100644 index 0000000..ebe3a86 --- /dev/null +++ b/main.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "path/filepath" + + "github.com/gofiber/fiber/v2" + cp "github.com/otiai10/copy" + "github.com/robfig/cron/v3" + "log" + "os" +) + +func main() { + if err := os.Mkdir("./public", 0750); err != nil && !os.IsExist(err) { + log.Fatal(err) + } + + app := fiber.New(fiber.Config{}) + + port := os.Getenv("PORT") + if port == "" { + port = "8989" + } + + c := cron.New() + + cronsetting := os.Getenv("CRONSETTING") + + if cronsetting == "" { + cronsetting = "0 1 * * *" // At 01:00, every day + } + + c.AddFunc(cronsetting, func() { + update() + }) + + c.Start() + + doupdate := os.Getenv("UPDATE") + + if doupdate != "" { + update() + } + + app.Static("/", "./public") + log.Fatalln(app.Listen(fmt.Sprintf(":%v", port))) +} + +func update() { + log.Println("Update") + + if err := os.Mkdir("./downloads", 0750); err != nil && !os.IsExist(err) { + log.Fatal(err) + } + + log.Println(filepath.Base("https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt")) + + urls := []string{"youtube", "abuse", "ads", "crypto", "drugs", "facebook", "fraud", "gambling", "malware", "phishing", "piracy", "porn", "ransomware", "redirect", "scam", "torrent", "tracking"} + + for uc := range urls { + log.Println("Updating", urls[uc]) + + url := fmt.Sprintf("https://blocklistproject.github.io/Lists/%s.txt", urls[uc]) + path := fmt.Sprintf("./downloads/%s.txt", urls[uc]) + log.Println("Downloading", url) + dlerr := DownloadFile(path, url) + + if dlerr != nil { + log.Println("Error downloading", dlerr) + } + } + + otherList := []string{"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt", "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", "https://adaway.org/hosts.txt", "https://v.firebog.net/hosts/static/w3kbl.txt", "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt", "https://block.energized.pro/unified/formats/domains.txt", "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn/hosts"} + + for oc := range otherList { + log.Println("Updating", otherList[oc]) + url := otherList[oc] + path := fmt.Sprintf("./downloads/%s", filepath.Base(url)) + log.Println("Downloading", url) + dlerr := DownloadFile(path, url) + + if dlerr != nil { + log.Println("Error downloading", dlerr) + } + + } + + copyerror := cp.Copy("./downloads", "./public") + + if copyerror != nil { + log.Println("Error copying files", copyerror) + } + +} + +func DownloadFile(filepath string, url string) error { + + // Get the data + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + log.Println("Sattus", resp.Status) + + if resp.Status == "200 OK" { + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + return err + + } + return err +}