Init
This commit is contained in:
Martin Donnelly
commit
111bb88d18
106
.gitignore
vendored
Normal file
106
.gitignore
vendored
Normal file
@ -0,0 +1,106 @@
|
||||
### Go template
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
go-blocklist
|
||||
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
### GoLand template
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# AWS User-specific
|
||||
.idea/**/aws.xml
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Gradle and Maven with auto-import
|
||||
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||
# since they will be recreated, and may cause churn. Uncomment if using
|
||||
# auto-import.
|
||||
# .idea/artifacts
|
||||
# .idea/compiler.xml
|
||||
# .idea/jarRepositories.xml
|
||||
# .idea/modules.xml
|
||||
# .idea/*.iml
|
||||
# .idea/modules
|
||||
# *.iml
|
||||
# *.ipr
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# SonarLint plugin
|
||||
.idea/sonarlint/
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
/downloads/
|
||||
/public/
|
||||
35
Makefile
Normal file
35
Makefile
Normal file
@ -0,0 +1,35 @@
|
||||
PROJECT = go-blocklist
|
||||
|
||||
VERSION="05"
|
||||
BUILD=`date +%yw%Vb`
|
||||
|
||||
ECR_REPO = git.caliban.io/martin
|
||||
|
||||
APP_IMAGE = $(ECR_REPO)/$(PROJECT):$(BUILD)$(VERSION)
|
||||
|
||||
NO_CACHE = true
|
||||
|
||||
.PHONY: build
|
||||
build:
|
||||
|
||||
GCO_ENABLED=0 GOOS=linux go build -o ${PROJECT} main.go
|
||||
|
||||
# docker build ./docker/. -t $(APP_IMAGE) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE) --compress=true
|
||||
docker build --platform linux/amd64 --no-cache -force-rm --tag ${APP_IMAGE} --file ./docker/Dockerfile .
|
||||
|
||||
|
||||
#push docker image to registry
|
||||
.PHONY: push
|
||||
push: build
|
||||
docker push $(APP_IMAGE)
|
||||
|
||||
|
||||
#push docker image to registry
|
||||
.PHONY: run
|
||||
run: build
|
||||
docker run $(APP_IMAGE)
|
||||
|
||||
ver:
|
||||
@echo '$(VERSION)'
|
||||
#echo $ERSION
|
||||
.PHONY: ver
|
||||
13
docker-compose.yml
Normal file
13
docker-compose.yml
Normal file
@ -0,0 +1,13 @@
|
||||
version: '3.5'
|
||||
|
||||
services:
|
||||
blocklist:
|
||||
container_name: blocklist
|
||||
image: git.caliban.io/martin/go-blocklist:24w36b05
|
||||
ports:
|
||||
- '8989:8989'
|
||||
restart: always
|
||||
volumes:
|
||||
- ./public:/app/public
|
||||
- ./downloads:/app/downloads
|
||||
|
||||
20
docker/Dockerfile
Normal file
20
docker/Dockerfile
Normal file
@ -0,0 +1,20 @@
|
||||
# FROM debian:12-slim
|
||||
FROM alpine:latest
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN mkdir -p /app/downloads /app/public
|
||||
|
||||
COPY ./go-blocklist /app/
|
||||
|
||||
|
||||
|
||||
# Need the following to get a go app to run inside a docker container
|
||||
# as per: https://www.fairlyusefulcode.co.uk/post/go-alpine-linux/
|
||||
RUN mkdir /lib64 && ln -s /lib/libc.musl-x86_64.so.1 /lib64/ld-linux-x86-64.so.2
|
||||
RUN chmod +x /app/go-blocklist
|
||||
|
||||
EXPOSE 8989
|
||||
|
||||
CMD [ "/app/go-blocklist"]
|
||||
|
||||
21
go.mod
Normal file
21
go.mod
Normal file
@ -0,0 +1,21 @@
|
||||
module go-blocklist
|
||||
|
||||
go 1.22
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.0.5 // indirect
|
||||
github.com/gofiber/fiber/v2 v2.52.5 // indirect
|
||||
github.com/google/uuid v1.5.0 // indirect
|
||||
github.com/klauspost/compress v1.17.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/otiai10/copy v1.14.0 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.0 // indirect
|
||||
github.com/valyala/bytebufferpool v1.0.0 // indirect
|
||||
github.com/valyala/fasthttp v1.51.0 // indirect
|
||||
github.com/valyala/tcplisten v1.0.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/sys v0.15.0 // indirect
|
||||
)
|
||||
33
go.sum
Normal file
33
go.sum
Normal file
@ -0,0 +1,33 @@
|
||||
github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs=
|
||||
github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/gofiber/fiber/v2 v2.52.5 h1:tWoP1MJQjGEe4GB5TUGOi7P2E0ZMMRx5ZTG4rT+yGMo=
|
||||
github.com/gofiber/fiber/v2 v2.52.5/go.mod h1:KEOE+cXMhXG0zHc9d8+E38hoX+ZN7bhOtgeF2oT6jrQ=
|
||||
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
|
||||
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM=
|
||||
github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU=
|
||||
github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w=
|
||||
github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
|
||||
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||
github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
|
||||
github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA=
|
||||
github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g=
|
||||
github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8=
|
||||
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
|
||||
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
|
||||
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
123
main.go
Normal file
123
main.go
Normal file
@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/gofiber/fiber/v2"
|
||||
cp "github.com/otiai10/copy"
|
||||
"github.com/robfig/cron/v3"
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := os.Mkdir("./public", 0750); err != nil && !os.IsExist(err) {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
app := fiber.New(fiber.Config{})
|
||||
|
||||
port := os.Getenv("PORT")
|
||||
if port == "" {
|
||||
port = "8989"
|
||||
}
|
||||
|
||||
c := cron.New()
|
||||
|
||||
cronsetting := os.Getenv("CRONSETTING")
|
||||
|
||||
if cronsetting == "" {
|
||||
cronsetting = "0 1 * * *" // At 01:00, every day
|
||||
}
|
||||
|
||||
c.AddFunc(cronsetting, func() {
|
||||
update()
|
||||
})
|
||||
|
||||
c.Start()
|
||||
|
||||
doupdate := os.Getenv("UPDATE")
|
||||
|
||||
if doupdate != "" {
|
||||
update()
|
||||
}
|
||||
|
||||
app.Static("/", "./public")
|
||||
log.Fatalln(app.Listen(fmt.Sprintf(":%v", port)))
|
||||
}
|
||||
|
||||
func update() {
|
||||
log.Println("Update")
|
||||
|
||||
if err := os.Mkdir("./downloads", 0750); err != nil && !os.IsExist(err) {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Println(filepath.Base("https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt"))
|
||||
|
||||
urls := []string{"youtube", "abuse", "ads", "crypto", "drugs", "facebook", "fraud", "gambling", "malware", "phishing", "piracy", "porn", "ransomware", "redirect", "scam", "torrent", "tracking"}
|
||||
|
||||
for uc := range urls {
|
||||
log.Println("Updating", urls[uc])
|
||||
|
||||
url := fmt.Sprintf("https://blocklistproject.github.io/Lists/%s.txt", urls[uc])
|
||||
path := fmt.Sprintf("./downloads/%s.txt", urls[uc])
|
||||
log.Println("Downloading", url)
|
||||
dlerr := DownloadFile(path, url)
|
||||
|
||||
if dlerr != nil {
|
||||
log.Println("Error downloading", dlerr)
|
||||
}
|
||||
}
|
||||
|
||||
otherList := []string{"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt", "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt", "https://adaway.org/hosts.txt", "https://v.firebog.net/hosts/static/w3kbl.txt", "https://raw.githubusercontent.com/kboghdady/youTube_ads_4_pi-hole/master/youtubelist.txt", "https://block.energized.pro/unified/formats/domains.txt", "https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews-gambling-porn/hosts"}
|
||||
|
||||
for oc := range otherList {
|
||||
log.Println("Updating", otherList[oc])
|
||||
url := otherList[oc]
|
||||
path := fmt.Sprintf("./downloads/%s", filepath.Base(url))
|
||||
log.Println("Downloading", url)
|
||||
dlerr := DownloadFile(path, url)
|
||||
|
||||
if dlerr != nil {
|
||||
log.Println("Error downloading", dlerr)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
copyerror := cp.Copy("./downloads", "./public")
|
||||
|
||||
if copyerror != nil {
|
||||
log.Println("Error copying files", copyerror)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func DownloadFile(filepath string, url string) error {
|
||||
|
||||
// Get the data
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Println("Sattus", resp.Status)
|
||||
|
||||
if resp.Status == "200 OK" {
|
||||
out, err := os.Create(filepath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
return err
|
||||
|
||||
}
|
||||
return err
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user