From a022ea60ea35a04f860e81c46a9f3ed7bcbc3e55 Mon Sep 17 00:00:00 2001 From: sajadMRjl Date: Wed, 28 Jan 2026 15:09:13 +0330 Subject: [PATCH] fix: easier import proxies --- .gitignore | 2 +- README.md | 47 ++++++++++++--- cmd/main.go | 98 +++++++++++++++---------------- data/valid_proxies.txt | 8 +++ internal/config/config.go | 1 + internal/sink/writer.go | 28 ++++++++- internal/source/loader.go | 118 +++++++++++++++++++++++++++++--------- 7 files changed, 214 insertions(+), 88 deletions(-) create mode 100644 data/valid_proxies.txt diff --git a/.gitignore b/.gitignore index d9dca24..5ce5cb0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,8 @@ scanner *.dll *.so *.dylib -data/ data/* +!data/valid_proxies.txt bin/ bin/* diff --git a/README.md b/README.md index 9b4634d..84bc258 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ +Here is the updated `README.md` incorporating the new features (Smart Input, Dual Output) and configuration options. + # Find Me Internet 🌐 -**Find Me Internet** is a high-performance, scalable proxy scanner and tester written in Go. It is designed to ingest thousands of proxy links (VLESS, VMess, Trojan, Reality, etc.), filter out dead nodes efficiently using "cheap" network checks, and rigorously test survivors using a real **Sing-box** core. +**Find Me Internet** is a high-performance, scalable proxy scanner and tester written in Go. It is designed to ingest thousands of proxy links (VLESS, VMess, Trojan, Reality, etc.) from files or URLs, filter out dead nodes efficiently using "cheap" network checks, and rigorously test survivors using a real **Sing-box** core. It uses a "Funnel Architecture" to minimize resource usage, allowing it to scan 100,000+ proxies without crashing your system. @@ -19,6 +21,7 @@ graph LR E -- Latency OK --> F(GeoIP Enricher) E -- Fail --> X F --> G[JSONL Sink] + F --> H[TXT Sink] ``` @@ -69,32 +72,58 @@ TEST_URL=http://cp.cloudflare.com # The target to fetch # --- Paths --- SING_BOX_PATH=./bin/sing-box -INPUT_PATH=./data/proxies.txt -OUTPUT_PATH=./data/valid_proxies.jsonl +INPUT_PATH=./data/proxies.txt # Default input source +OUTPUT_PATH=./data/valid_proxies.jsonl # Detailed output +TXT_OUTPUT_PATH=./data/valid_proxies.txt # Simple list output GEOIP_PATH=./data/GeoLite2-Country.mmdb ``` ## 🏃 Usage -1. **Prepare Input:** - Add your raw proxy links (one per line) to `data/proxies.txt`. -2. **Run the Scanner:** +You can run the scanner in three ways: + +### 1. Default Mode + +Reads the input path defined in your `.env` file (`INPUT_PATH`). ```bash go run cmd/main.go ``` -3. **View Results:** - Valid proxies are streamed to `data/valid_proxies.jsonl`. +### 2. File Override + +Pass a specific local file path as an argument. ```bash +go run cmd/main.go ./my_new_proxies.txt + +``` + +### 3. URL Streaming + +Pass a URL (starting with `http://` or `https://`) to fetch and scan a remote subscription directly. + +```bash +go run cmd/main.go https://raw.githubusercontent.com/example/proxies/main/list.txt + +``` + +## 📊 Viewing Results + +The scanner generates two output files in your `data/` folder: + +1. **valid_proxies.txt**: A clean list of working proxy links, ready to copy-paste into V2Ray clients. +2. **valid_proxies.jsonl**: Detailed JSON logs for analysis. + +```bash +# Watch results in real-time tail -f data/valid_proxies.jsonl ``` -### Output Example +### Output Example (JSONL) ```json {"link":"vless://uuid@1.2.3.4:443...","type":"vless","address":"1.2.3.4","port":443,"latency_ms":145,"country":"DE"} diff --git a/cmd/main.go b/cmd/main.go index 7fe17c4..1a39eb7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -23,44 +23,57 @@ func main() { cfg := config.Load() logger.Setup(cfg.LogLevel) - // Graceful Shutdown Channel + // CLI Argument Override + // Usage: ./find-me-internet [OPTIONAL_INPUT_SOURCE] + if len(os.Args) > 1 { + cfg.InputPath = os.Args[1] + slog.Info("input_source_overridden", "source", cfg.InputPath) + } + sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) // 2. Services geoDB, err := geoip.Open(cfg.GeoIPPath) if err != nil { - slog.Warn("geoip_db_missing", "error", err, "msg", "Countries will be marked N/A") + slog.Warn("geoip_db_missing", "error", err) } else { defer geoDB.Close() } - resultsWriter, err := sink.NewJSONL(cfg.OutputPath) + jsonWriter, err := sink.NewJSONL(cfg.OutputPath) if err != nil { - slog.Error("cannot_create_output_file", "error", err) + slog.Error("cannot_create_json_output", "error", err) os.Exit(1) } - defer resultsWriter.Close() + defer jsonWriter.Close() + + txtWriter, err := sink.NewText(cfg.TxtOutputPath) + if err != nil { + slog.Error("cannot_create_txt_output", "error", err) + os.Exit(1) + } + defer txtWriter.Close() deduplicator := dedup.New() netFilter := filter.NewPipeline(cfg.TcpTimeout) boxRunner := tester.NewRunner(cfg.SingBoxPath, cfg.TestURL, cfg.TestTimeout) - // 3. Input Stream - linkStream, err := source.LoadFromFile(cfg.InputPath) + // 3. Input Stream (Smart Load) + // Supports both http://... and ./path/to/file.txt + linkStream, err := source.Load(cfg.InputPath) if err != nil { - slog.Error("input_source_failed", "error", err) + slog.Error("input_source_failed", "error", err, "path", cfg.InputPath) os.Exit(1) } - // 4. Worker Pool & Counters + // 4. Worker Pool var wg sync.WaitGroup semaphore := make(chan struct{}, cfg.Workers) - // Counters for final stats countProcessed := 0 countValid := 0 - var mu sync.Mutex // Protects countValid from race conditions + var mu sync.Mutex slog.Info("pipeline_started", "workers", cfg.Workers) @@ -72,70 +85,57 @@ loop: slog.Info("shutdown_signal_received", "msg", "finishing pending jobs...") break loop default: - // Continue } wg.Add(1) go func(raw string) { defer wg.Done() - // --- STAGE 1: PARSE --- + // A. Parse proxy, err := parser.ParseLink(raw) - if err != nil { - return - } + if err != nil { return } - // --- STAGE 2: DEDUP --- - if deduplicator.Seen(proxy.Address, proxy.Port) { - return // Skip duplicates silently - } + // B. Dedup + if deduplicator.Seen(proxy.Address, proxy.Port) { return } - // --- STAGE 3: FILTER --- - if !netFilter.Check(proxy) { - return - } + // C. Filter + if !netFilter.Check(proxy) { return } - // --- STAGE 4: TEST --- - semaphore <- struct{}{} // Rate limit expensive tests + // D. Test + semaphore <- struct{}{} err = boxRunner.Test(proxy) <-semaphore - if err != nil { - return - } + if err != nil { return } - // --- STAGE 5: ENRICH --- + // E. Enrich if geoDB != nil { proxy.Country = geoDB.Lookup(proxy.Address) } - // --- STAGE 6: SAVE --- - if err := resultsWriter.Write(proxy); err != nil { - slog.Error("write_failed", "error", err) - } else { - // Thread-Safe Increment - mu.Lock() - countValid++ - mu.Unlock() + // F. Save + jsonWriter.Write(proxy) + txtWriter.Write(proxy) - slog.Info("proxy_saved", - "country", proxy.Country, - "latency", proxy.Latency.Milliseconds(), - "type", proxy.Type, - ) - } + // Stats + mu.Lock() + countValid++ + mu.Unlock() + + slog.Info("proxy_saved", + "country", proxy.Country, + "latency", proxy.Latency.Milliseconds(), + "type", proxy.Type, + ) }(rawLink) countProcessed++ if countProcessed % 1000 == 0 { - slog.Info("progress_report", "processed", countProcessed, "valid_so_far", countValid) + slog.Info("progress_report", "processed", countProcessed, "valid", countValid) } } wg.Wait() - slog.Info("scan_finished", - "total_processed", countProcessed, - "total_valid", countValid, - ) + slog.Info("scan_finished", "total_processed", countProcessed, "total_valid", countValid) } \ No newline at end of file diff --git a/data/valid_proxies.txt b/data/valid_proxies.txt new file mode 100644 index 0000000..ffeb120 --- /dev/null +++ b/data/valid_proxies.txt @@ -0,0 +1,8 @@ +vless://9e685fe3-e0f9-482d-939c-200a3f89b363@172.64.145.38:8443?path=%2F%3Fed%3D2560fp%3Drandom&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=vyznthvt7f5fr.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=vyznthvt7f5fr.zjde5.de5.net#%F0%9F%87%A9%F0%9F%87%AA%20%40vmesspv +vless://bb8c74a1-abc1-4511-b100-9876e30cb65c@172.64.145.38:8443?path=%2F%3Fed%3D2560&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=xfjd79v2tjscrm6jqo.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=xfjd79v2tjscrm6jqo.zjde5.de5.net#@Vip_Security join us - 55 +vless://f85f60b1-2b96-49e9-8bde-b656d1516df0@104.17.165.123:8443?path=%2F%3Fed%3D2560&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=gx8rw8fz783ncefn332y7uyfsvb59o820mryrxu1cj19jiuuur.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=gx8rw8fz783ncefn332y7uyfsvb59o820mryrxu1cj19jiuuur.zjde5.de5.net#@Vip_Security join us - 67 +vless://4525c260-df3c-4f62-b8f1-f4f5f305694b@104.17.164.123:8443?path=%2F%3Fed%3D2560&security=tls&encryption=none&insecure=0&host=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net#%F0%9F%8C%8E%20%40vmesspv +vless://9e685fe3-e0f9-482d-939c-200a3f89b363@172.64.145.38:8443?path=%2F%3Fed%3D2560fp%3Drandom&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=vyznthvt7f5fr.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=vyznthvt7f5fr.zjde5.de5.net#%F0%9F%87%A9%F0%9F%87%AA%20%40vmesspv +vless://4525c260-df3c-4f62-b8f1-f4f5f305694b@66.81.247.155:443?path=%2F%3Fed%3D512&security=tls&encryption=none&insecure=0&host=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net#%F0%9F%8C%8E%20%40vmesspv +vless://83f03646-fb28-44cc-9d2c-8853f6c09285@104.17.162.123:8443?path=%2F%3Fed%3D%23TELEGRAM-Yam%3Fed%3D512&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=r4fnviw9jl4i4rx.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=r4fnviw9jl4i4rx.zjde5.de5.net#@chthxyz - 61 +vless://3a4ddfac-e7da-48c9-9648-4a366109fc3a@api.steamsale.ir:443?path=%2FX0PX5Vup1qlVVzhxp6ic50a&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=mtn.vpsmee.ir&fp=chrome&type=ws&allowInsecure=0&sni=cdn.vpsmee.ir#@chthxyz - 28 diff --git a/internal/config/config.go b/internal/config/config.go index 8f25bd4..00c86d7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,6 +23,7 @@ type Config struct { InputPath string `envconfig:"INPUT_PATH" default:"proxies.txt"` OutputPath string `envconfig:"OUTPUT_PATH" default:"valid.jsonl"` GeoIPPath string `envconfig:"GEOIP_PATH" default:"GeoLite2-Country.mmdb"` + TxtOutputPath string `envconfig:"TXT_OUTPUT_PATH" default:"valid.txt"` } // Load reads .env and processes environment variables diff --git a/internal/sink/writer.go b/internal/sink/writer.go index 22c088e..f4b2cf1 100644 --- a/internal/sink/writer.go +++ b/internal/sink/writer.go @@ -35,4 +35,30 @@ func (w *JSONLWriter) Write(p *model.Proxy) error { func (w *JSONLWriter) Close() { w.file.Close() -} \ No newline at end of file +} + + +type TextWriter struct { + file *os.File + mu sync.Mutex +} + +func NewText(path string) (*TextWriter, error) { + // We use O_APPEND | O_CREATE | O_WRONLY + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return nil, err + } + return &TextWriter{file: f}, nil +} + +func (w *TextWriter) Write(p *model.Proxy) error { + w.mu.Lock() + defer w.mu.Unlock() + + // Just write the RawLink + NewLine + _, err := w.file.WriteString(p.RawLink + "\n") + return err +} + +func (w *TextWriter) Close() { w.file.Close() } \ No newline at end of file diff --git a/internal/source/loader.go b/internal/source/loader.go index 6bf3089..7fd44fe 100644 --- a/internal/source/loader.go +++ b/internal/source/loader.go @@ -2,60 +2,122 @@ package source import ( "bufio" + "encoding/base64" + "io" + "log/slog" "net/http" "os" "strings" + "sync" + "time" ) -// Loader returns a channel of strings to keep memory usage low -func LoadFromFile(path string) (<-chan string, error) { - file, err := os.Open(path) - if err != nil { - return nil, err +// Load is the entry point. It determines if the input is a URL or a File. +func Load(input string) (<-chan string, error) { + input = strings.TrimSpace(input) + out := make(chan string) + + // If the input arg itself is a URL, fetch it directly + if strings.HasPrefix(input, "http://") || strings.HasPrefix(input, "https://") { + slog.Info("loading_from_remote_url", "url", input) + go func() { + defer close(out) + fetchAndStream(input, out) + }() + return out, nil } + // Otherwise, treat it as a file (which might contain URLs!) + slog.Info("loading_from_file", "path", input) + return loadFromFileRecursive(input), nil +} + +// loadFromFileRecursive reads a file line-by-line. +// If a line is a URL, it fetches it. If it's a proxy, it sends it. +func loadFromFileRecursive(path string) <-chan string { out := make(chan string) go func() { - defer file.Close() defer close(out) + file, err := os.Open(path) + if err != nil { + slog.Error("file_open_failed", "path", path, "error", err) + return + } + defer file.Close() + + var wg sync.WaitGroup scanner := bufio.NewScanner(file) - // Increase buffer size for very long lines (some subscription links are huge) - buf := make([]byte, 0, 64*1024) + buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, 1024*1024) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - if line != "" && !strings.HasPrefix(line, "#") { + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { + continue + } + + // MAGIC: If the line inside the file is a URL, fetch it! + if strings.HasPrefix(line, "http://") || strings.HasPrefix(line, "https://") { + wg.Add(1) + // Fetch in parallel + go func(url string) { + defer wg.Done() + fetchAndStream(url, out) + }(line) + } else { + // It's just a raw proxy link out <- line } } + + // Wait for all subscription fetches to finish before closing channel + wg.Wait() }() - return out, nil + return out } -// LoadFromURL streams directly from a URL (e.g., Github raw) -func LoadFromURL(url string) (<-chan string, error) { - resp, err := http.Get(url) +// fetchAndStream downloads content from a URL and parses it +func fetchAndStream(url string, out chan<- string) { + client := &http.Client{Timeout: 15 * time.Second} + resp, err := client.Get(url) if err != nil { - return nil, err + slog.Warn("subscription_fetch_failed", "url", url, "error", err) + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + slog.Warn("subscription_bad_status", "url", url, "status", resp.StatusCode) + return } - out := make(chan string) - go func() { - defer resp.Body.Close() - defer close(out) - - scanner := bufio.NewScanner(resp.Body) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if line != "" { - out <- line - } - } - }() + // Read full body to handle Base64 decoding if necessary + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return + } + content := string(bodyBytes) - return out, nil + // STEP 1: Try Base64 Decode + // Many subscriptions return a base64 blob of proxies + if decoded, err := base64.StdEncoding.DecodeString(content); err == nil { + content = string(decoded) + } else if decoded, err := base64.RawStdEncoding.DecodeString(content); err == nil { + content = string(decoded) + } + + // STEP 2: Split by Newline and Stream + lines := strings.Split(content, "\n") + count := 0 + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "#") { + out <- line + count++ + } + } + slog.Info("subscription_loaded", "url", url, "proxies_found", count) } \ No newline at end of file