Init
This commit is contained in:
Martin Donnelly
commit
111bb88d18
106
.gitignore
vendored
Normal file
106
.gitignore
vendored
Normal file
@ -0,0 +1,106 @@
|
|||||||
|
### Go template
|
||||||
|
# If you prefer the allow list template instead of the deny list, see community template:
|
||||||
|
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||||
|
#
|
||||||
|
# Binaries for programs and plugins
|
||||||
|
go-blocklist
|
||||||
|
|
||||||
|
*.exe
|
||||||
|
*.exe~
|
||||||
|
*.dll
|
||||||
|
*.so
|
||||||
|
*.dylib
|
||||||
|
|
||||||
|
# Test binary, built with `go test -c`
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||||
|
*.out
|
||||||
|
|
||||||
|
# Dependency directories (remove the comment below to include it)
|
||||||
|
# vendor/
|
||||||
|
|
||||||
|
# Go workspace file
|
||||||
|
go.work
|
||||||
|
|
||||||
|
### GoLand template
|
||||||
|
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||||
|
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||||
|
|
||||||
|
# User-specific stuff
|
||||||
|
.idea/**/workspace.xml
|
||||||
|
.idea/**/tasks.xml
|
||||||
|
.idea/**/usage.statistics.xml
|
||||||
|
.idea/**/dictionaries
|
||||||
|
.idea/**/shelf
|
||||||
|
|
||||||
|
# AWS User-specific
|
||||||
|
.idea/**/aws.xml
|
||||||
|
|
||||||
|
# Generated files
|
||||||
|
.idea/**/contentModel.xml
|
||||||
|
|
||||||
|
# Sensitive or high-churn files
|
||||||
|
.idea/**/dataSources/
|
||||||
|
.idea/**/dataSources.ids
|
||||||
|
.idea/**/dataSources.local.xml
|
||||||
|
.idea/**/sqlDataSources.xml
|
||||||
|
.idea/**/dynamic.xml
|
||||||
|
.idea/**/uiDesigner.xml
|
||||||
|
.idea/**/dbnavigator.xml
|
||||||
|
|
||||||
|
# Gradle
|
||||||
|
.idea/**/gradle.xml
|
||||||
|
.idea/**/libraries
|
||||||
|
|
||||||
|
# Gradle and Maven with auto-import
|
||||||
|
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||||
|
# since they will be recreated, and may cause churn. Uncomment if using
|
||||||
|
# auto-import.
|
||||||
|
# .idea/artifacts
|
||||||
|
# .idea/compiler.xml
|
||||||
|
# .idea/jarRepositories.xml
|
||||||
|
# .idea/modules.xml
|
||||||
|
# .idea/*.iml
|
||||||
|
# .idea/modules
|
||||||
|
# *.iml
|
||||||
|
# *.ipr
|
||||||
|
|
||||||
|
# CMake
|
||||||
|
cmake-build-*/
|
||||||
|
|
||||||
|
# Mongo Explorer plugin
|
||||||
|
.idea/**/mongoSettings.xml
|
||||||
|
|
||||||
|
# File-based project format
|
||||||
|
*.iws
|
||||||
|
|
||||||
|
# IntelliJ
|
||||||
|
out/
|
||||||
|
|
||||||
|
# mpeltonen/sbt-idea plugin
|
||||||
|
.idea_modules/
|
||||||
|
|
||||||
|
# JIRA plugin
|
||||||
|
atlassian-ide-plugin.xml
|
||||||
|
|
||||||
|
# Cursive Clojure plugin
|
||||||
|
.idea/replstate.xml
|
||||||
|
|
||||||
|
# SonarLint plugin
|
||||||
|
.idea/sonarlint/
|
||||||
|
|
||||||
|
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||||
|
com_crashlytics_export_strings.xml
|
||||||
|
crashlytics.properties
|
||||||
|
crashlytics-build.properties
|
||||||
|
fabric.properties
|
||||||
|
|
||||||
|
# Editor-based Rest Client
|
||||||
|
.idea/httpRequests
|
||||||
|
|
||||||
|
# Android studio 3.1+ serialized cache file
|
||||||
|
.idea/caches/build_file_checksums.ser
|
||||||
|
|
||||||
|
/downloads/
|
||||||
|
/public/
|
||||||
35
Makefile
Normal file
35
Makefile
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
PROJECT = go-blocklist
|
||||||
|
|
||||||
|
VERSION="05"
|
||||||
|
BUILD=`date +%yw%Vb`
|
||||||
|
|
||||||
|
ECR_REPO = git.caliban.io/martin
|
||||||
|
|
||||||
|
APP_IMAGE = $(ECR_REPO)/$(PROJECT):$(BUILD)$(VERSION)
|
||||||
|
|
||||||
|
NO_CACHE = true
|
||||||
|
|
||||||
|
.PHONY: build
|
||||||
|
build:
|
||||||
|
|
||||||
|
GCO_ENABLED=0 GOOS=linux go build -o ${PROJECT} main.go
|
||||||
|
|
||||||
|
# docker build ./docker/. -t $(APP_IMAGE) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE) --compress=true
|
||||||
|
docker build --platform linux/amd64 --no-cache -force-rm --tag ${APP_IMAGE} --file ./docker/Dockerfile .
|
||||||
|
|
||||||
|
|
||||||
|
#push docker image to registry
|
||||||
|
.PHONY: push
|
||||||
|
push: build
|
||||||
|
docker push $(APP_IMAGE)
|
||||||
|
|
||||||
|
|
||||||
|
#push docker image to registry
|
||||||
|
.PHONY: run
|
||||||
|
run: build
|
||||||
|
docker run $(APP_IMAGE)
|
||||||
|
|
||||||
|
ver:
|
||||||
|
@echo '$(VERSION)'
|
||||||
|
#echo $ERSION
|
||||||
|
.PHONY: ver
|
||||||
13
docker-compose.yml
Normal file
13
docker-compose.yml
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
version: '3.5'
|
||||||
|
|
||||||
|
services:
|
||||||
|
blocklist:
|
||||||
|
container_name: blocklist
|
||||||
|
image: git.caliban.io/martin/go-blocklist:24w36b05
|
||||||
|
ports:
|
||||||
|
- '8989:8989'
|
||||||
|
restart: always
|
||||||
|
volumes:
|
||||||
|
- ./public:/app/public
|
||||||
|
- ./downloads:/app/downloads
|
||||||
|
|
||||||
20
docker/Dockerfile
Normal file
20
docker/Dockerfile
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
# FROM debian:12-slim
|
||||||
|
FROM alpine:latest
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
RUN mkdir -p /app/downloads /app/public
|
||||||
|
|
||||||
|
COPY ./go-blocklist /app/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Need the following to get a go app to run inside a docker container
|
||||||
|
# as per: https://www.fairlyusefulcode.co.uk/post/go-alpine-linux/
|
||||||
|
RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
|
||||||
|
RUN chmod +x /app/go-blocklist
|
||||||
|
|
||||||
|
EXPOSE 8989
|
||||||
|
|
||||||
|
CMD [ "/app/go-blocklist"]
|
||||||
|
|
||||||
21
go.mod
Normal file
21
go.mod
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
module go-blocklist
|
||||||
|
|
||||||
|
go 1.22
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.5 // indirect
|
||||||
|
github.com/google/uuid v1.5.0 // indirect
|
||||||
|
github.com/klauspost/compress v1.17.0 // indirect
|
||||||
|
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||||
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
|
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||||
|
github.com/otiai10/copy v1.14.0 // indirect
|
||||||
|
github.com/rivo/uniseg v0.2.0 // indirect
|
||||||
|
github.com/robfig/cron/v3 v3.0.0 // indirect
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||||
|
github.com/valyala/fasthttp v1.51.0 // indirect
|
||||||
|
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||||
|
golang.org/x/sync v0.3.0 // indirect
|
||||||
|
golang.org/x/sys v0.15.0 // indirect
|
||||||
|
)
|
||||||
33
go.sum
Normal file
33
go.sum
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||||
|
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
|
||||||
|
github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||||
|
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
|
||||||
|
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
|
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
|
||||||
|
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||||
|
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||||
|
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||||
|
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||||
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
|
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||||
|
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||||
|
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
|
||||||
|
github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
|
||||||
|
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||||
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
|
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
|
||||||
|
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||||
|
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||||
|
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
|
||||||
|
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
|
||||||
|
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||||
|
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||||
|
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||||
|
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||||
|
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||||
|
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
123
main.go
Normal file
123
main.go
Normal file
@ -0,0 +1,123 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/gofiber/fiber/v2"
|
||||||
|
cp "github.com/otiai10/copy"
|
||||||
|
"github.com/robfig/cron/v3"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := os.Mkdir("./public", 0750); err != nil && !os.IsExist(err) {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
app := fiber.New(fiber.Config{})
|
||||||
|
|
||||||
|
port := os.Getenv("PORT")
|
||||||
|
if port == "" {
|
||||||
|
port = "8989"
|
||||||
|
}
|
||||||
|
|
||||||
|
c := cron.New()
|
||||||
|
|
||||||
|
cronsetting := os.Getenv("CRONSETTING")
|
||||||
|
|
||||||
|
if cronsetting == "" {
|
||||||
|
cronsetting = "0 1 * * *" // At 01:00, every day
|
||||||
|
}
|
||||||
|
|
||||||
|
c.AddFunc(cronsetting, func() {
|
||||||
|
update()
|
||||||
|
})
|
||||||
|
|
||||||
|
c.Start()
|
||||||
|
|
||||||
|
doupdate := os.Getenv("UPDATE")
|
||||||
|
|
||||||
|
if doupdate != "" {
|
||||||
|
update()
|
||||||
|
}
|
||||||
|
|
||||||
|
app.Static("/", "./public")
|
||||||
|
log.Fatalln(app.Listen(fmt.Sprintf(":%v", port)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func update() {
|
||||||
|
log.Println("Update")
|
||||||
|
|
||||||
|
if err := os.Mkdir("./downloads", 0750); err != nil && !os.IsExist(err) {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Println(filepath.Base("https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt"))
|
||||||
|
|
||||||
|
urls := []string{"youtube", "abuse", "ads", "crypto", "drugs", "facebook", "fraud", "gambling", "malware", "phishing", "piracy", "porn", "ransomware", "redirect", "scam", "torrent", "tracking"}
|
||||||
|
|
||||||
|
for uc := range urls {
|
||||||
|
log.Println("Updating", urls[uc])
|
||||||
|
|
||||||
|
url := fmt.Sprintf("https://blocklistproject.github.io/Lists/%s.txt", urls[uc])
|
||||||
|
path := fmt.Sprintf("./downloads/%s.txt", urls[uc])
|
||||||
|
log.Println("Downloading", url)
|
||||||
|
dlerr := DownloadFile(path, url)
|
||||||
|
|
||||||
|
if dlerr != nil {
|
||||||
|
log.Println("Error downloading", dlerr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
otherList := []string{"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt", "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", "https://adaway.org/hosts.txt", "https://v.firebog.net/hosts/static/w3kbl.txt", "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt", "https://block.energized.pro/unified/formats/domains.txt", "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn/hosts"}
|
||||||
|
|
||||||
|
for oc := range otherList {
|
||||||
|
log.Println("Updating", otherList[oc])
|
||||||
|
url := otherList[oc]
|
||||||
|
path := fmt.Sprintf("./downloads/%s", filepath.Base(url))
|
||||||
|
log.Println("Downloading", url)
|
||||||
|
dlerr := DownloadFile(path, url)
|
||||||
|
|
||||||
|
if dlerr != nil {
|
||||||
|
log.Println("Error downloading", dlerr)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
copyerror := cp.Copy("./downloads", "./public")
|
||||||
|
|
||||||
|
if copyerror != nil {
|
||||||
|
log.Println("Error copying files", copyerror)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func DownloadFile(filepath string, url string) error {
|
||||||
|
|
||||||
|
// Get the data
|
||||||
|
resp, err := http.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
log.Println("Sattus", resp.Status)
|
||||||
|
|
||||||
|
if resp.Status == "200 OK" {
|
||||||
|
out, err := os.Create(filepath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer out.Close()
|
||||||
|
|
||||||
|
// Write the body to file
|
||||||
|
_, err = io.Copy(out, resp.Body)
|
||||||
|
return err
|
||||||
|
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user