fix: easier import proxies

.gitignore

@@ -6,8 +6,8 @@ scanner
 *.dll
 *.so
 *.dylib
-data/
 data/*
+!data/valid_proxies.txt
 bin/
 bin/*
 

README.md

@@ -1,6 +1,8 @@
+Here is the updated `README.md` incorporating the new features (Smart Input, Dual Output) and configuration options.
+
 # Find Me Internet 🌐
 
-**Find Me Internet** is a high-performance, scalable proxy scanner and tester written in Go. It is designed to ingest thousands of proxy links (VLESS, VMess, Trojan, Reality, etc.), filter out dead nodes efficiently using "cheap" network checks, and rigorously test survivors using a real **Sing-box** core.
+**Find Me Internet** is a high-performance, scalable proxy scanner and tester written in Go. It is designed to ingest thousands of proxy links (VLESS, VMess, Trojan, Reality, etc.) from files or URLs, filter out dead nodes efficiently using "cheap" network checks, and rigorously test survivors using a real **Sing-box** core.
 
 It uses a "Funnel Architecture" to minimize resource usage, allowing it to scan 100,000+ proxies without crashing your system.
 
@@ -19,6 +21,7 @@ graph LR
     E -- Latency OK --> F(GeoIP Enricher)
     E -- Fail --> X
     F --> G[JSONL Sink]
+    F --> H[TXT Sink]
 
 ```
 
@@ -69,32 +72,58 @@ TEST_URL=http://cp.cloudflare.com  # The target to fetch
 
 # --- Paths ---
 SING_BOX_PATH=./bin/sing-box
-INPUT_PATH=./data/proxies.txt
-OUTPUT_PATH=./data/valid_proxies.jsonl
+INPUT_PATH=./data/proxies.txt            # Default input source
+OUTPUT_PATH=./data/valid_proxies.jsonl   # Detailed output
+TXT_OUTPUT_PATH=./data/valid_proxies.txt # Simple list output
 GEOIP_PATH=./data/GeoLite2-Country.mmdb
 
 ```
 
 ## 🏃 Usage
 
-1. **Prepare Input:**
-   Add your raw proxy links (one per line) to `data/proxies.txt`.
-2. **Run the Scanner:**
+You can run the scanner in three ways:
+
+### 1. Default Mode
+
+Reads the input path defined in your `.env` file (`INPUT_PATH`).
 
 ```bash
 go run cmd/main.go
 
 ```
 
-3. **View Results:**
-   Valid proxies are streamed to `data/valid_proxies.jsonl`.
+### 2. File Override
+
+Pass a specific local file path as an argument.
 
 ```bash
+go run cmd/main.go ./my_new_proxies.txt
+
+```
+
+### 3. URL Streaming
+
+Pass a URL (starting with `http://` or `https://`) to fetch and scan a remote subscription directly.
+
+```bash
+go run cmd/main.go https://raw.githubusercontent.com/example/proxies/main/list.txt
+
+```
+
+## 📊 Viewing Results
+
+The scanner generates two output files in your `data/` folder:
+
+1. **valid_proxies.txt**: A clean list of working proxy links, ready to copy-paste into V2Ray clients.
+2. **valid_proxies.jsonl**: Detailed JSON logs for analysis.
+
+```bash
+# Watch results in real-time
 tail -f data/valid_proxies.jsonl
 
 ```
 
-### Output Example
+### Output Example (JSONL)
 
 ```json
 {"link":"vless://uuid@1.2.3.4:443...","type":"vless","address":"1.2.3.4","port":443,"latency_ms":145,"country":"DE"}

cmd/main.go

@@ -23,44 +23,57 @@ func main() {
 	cfg := config.Load()
 	logger.Setup(cfg.LogLevel)
 	
-	// Graceful Shutdown Channel
+	// CLI Argument Override
+	// Usage: ./find-me-internet [OPTIONAL_INPUT_SOURCE]
+	if len(os.Args) > 1 {
+		cfg.InputPath = os.Args[1]
+		slog.Info("input_source_overridden", "source", cfg.InputPath)
+	}
+
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
 
 	// 2. Services
 	geoDB, err := geoip.Open(cfg.GeoIPPath)
 	if err != nil {
-		slog.Warn("geoip_db_missing", "error", err, "msg", "Countries will be marked N/A")
+		slog.Warn("geoip_db_missing", "error", err)
 	} else {
 		defer geoDB.Close()
 	}
 
-	resultsWriter, err := sink.NewJSONL(cfg.OutputPath)
+	jsonWriter, err := sink.NewJSONL(cfg.OutputPath)
 	if err != nil {
-		slog.Error("cannot_create_output_file", "error", err)
+		slog.Error("cannot_create_json_output", "error", err)
 		os.Exit(1)
 	}
-	defer resultsWriter.Close()
+	defer jsonWriter.Close()
+
+	txtWriter, err := sink.NewText(cfg.TxtOutputPath)
+	if err != nil {
+		slog.Error("cannot_create_txt_output", "error", err)
+		os.Exit(1)
+	}
+	defer txtWriter.Close()
 
 	deduplicator := dedup.New()
 	netFilter := filter.NewPipeline(cfg.TcpTimeout)
 	boxRunner := tester.NewRunner(cfg.SingBoxPath, cfg.TestURL, cfg.TestTimeout)
 
-	// 3. Input Stream
-	linkStream, err := source.LoadFromFile(cfg.InputPath)
+	// 3. Input Stream (Smart Load)
+	// Supports both http://... and ./path/to/file.txt
+	linkStream, err := source.Load(cfg.InputPath)
 	if err != nil {
-		slog.Error("input_source_failed", "error", err)
+		slog.Error("input_source_failed", "error", err, "path", cfg.InputPath)
 		os.Exit(1)
 	}
 
-	// 4. Worker Pool & Counters
+	// 4. Worker Pool
 	var wg sync.WaitGroup
 	semaphore := make(chan struct{}, cfg.Workers)
 	
-	// Counters for final stats
 	countProcessed := 0
 	countValid := 0
-	var mu sync.Mutex // Protects countValid from race conditions
+	var mu sync.Mutex
 	
 	slog.Info("pipeline_started", "workers", cfg.Workers)
 
@@ -72,70 +85,57 @@ loop:
 			slog.Info("shutdown_signal_received", "msg", "finishing pending jobs...")
 			break loop
 		default:
-			// Continue
 		}
 
 		wg.Add(1)
 		go func(raw string) {
 			defer wg.Done()
 			
-			// --- STAGE 1: PARSE ---
+			// A. Parse
 			proxy, err := parser.ParseLink(raw)
-			if err != nil {
-				return
-			}
+			if err != nil { return }
 
-			// --- STAGE 2: DEDUP ---
-			if deduplicator.Seen(proxy.Address, proxy.Port) {
-				return // Skip duplicates silently
-			}
+			// B. Dedup
+			if deduplicator.Seen(proxy.Address, proxy.Port) { return }
 
-			// --- STAGE 3: FILTER ---
-			if !netFilter.Check(proxy) {
-				return
-			}
+			// C. Filter
+			if !netFilter.Check(proxy) { return }
 
-			// --- STAGE 4: TEST ---
-			semaphore <- struct{}{} // Rate limit expensive tests
+			// D. Test
+			semaphore <- struct{}{}
 			err = boxRunner.Test(proxy)
 			<-semaphore
 
-			if err != nil {
-				return
-			}
+			if err != nil { return }
 
-			// --- STAGE 5: ENRICH ---
+			// E. Enrich
 			if geoDB != nil {
 				proxy.Country = geoDB.Lookup(proxy.Address)
 			}
 
-			// --- STAGE 6: SAVE ---
-			if err := resultsWriter.Write(proxy); err != nil {
-				slog.Error("write_failed", "error", err)
-			} else {
-				// Thread-Safe Increment
-				mu.Lock()
-				countValid++
-				mu.Unlock()
+			// F. Save
+			jsonWriter.Write(proxy)
+			txtWriter.Write(proxy)
 
-				slog.Info("proxy_saved", 
-					"country", proxy.Country, 
-					"latency", proxy.Latency.Milliseconds(),
-					"type", proxy.Type,
-				)
-			}
+			// Stats
+			mu.Lock()
+			countValid++
+			mu.Unlock()
+
+			slog.Info("proxy_saved", 
+				"country", proxy.Country, 
+				"latency", proxy.Latency.Milliseconds(),
+				"type", proxy.Type,
+			)
 
 		}(rawLink)
 		
 		countProcessed++
 		if countProcessed % 1000 == 0 {
-			slog.Info("progress_report", "processed", countProcessed, "valid_so_far", countValid)
+			slog.Info("progress_report", "processed", countProcessed, "valid", countValid)
 		}
 	}
 
 	wg.Wait()
-	slog.Info("scan_finished", 
-		"total_processed", countProcessed, 
-		"total_valid", countValid,
-	)
+	slog.Info("scan_finished", "total_processed", countProcessed, "total_valid", countValid)
 }

data/valid_proxies.txt

@@ -0,0 +1,8 @@
+vless://9e685fe3-e0f9-482d-939c-200a3f89b363@172.64.145.38:8443?path=%2F%3Fed%3D2560fp%3Drandom&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=vyznthvt7f5fr.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=vyznthvt7f5fr.zjde5.de5.net#%F0%9F%87%A9%F0%9F%87%AA%20%40vmesspv
+vless://bb8c74a1-abc1-4511-b100-9876e30cb65c@172.64.145.38:8443?path=%2F%3Fed%3D2560&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=xfjd79v2tjscrm6jqo.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=xfjd79v2tjscrm6jqo.zjde5.de5.net#@Vip_Security join us - 55
+vless://f85f60b1-2b96-49e9-8bde-b656d1516df0@104.17.165.123:8443?path=%2F%3Fed%3D2560&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=gx8rw8fz783ncefn332y7uyfsvb59o820mryrxu1cj19jiuuur.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=gx8rw8fz783ncefn332y7uyfsvb59o820mryrxu1cj19jiuuur.zjde5.de5.net#@Vip_Security join us - 67
+vless://4525c260-df3c-4f62-b8f1-f4f5f305694b@104.17.164.123:8443?path=%2F%3Fed%3D2560&security=tls&encryption=none&insecure=0&host=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net#%F0%9F%8C%8E%20%40vmesspv
+vless://9e685fe3-e0f9-482d-939c-200a3f89b363@172.64.145.38:8443?path=%2F%3Fed%3D2560fp%3Drandom&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=vyznthvt7f5fr.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=vyznthvt7f5fr.zjde5.de5.net#%F0%9F%87%A9%F0%9F%87%AA%20%40vmesspv
+vless://4525c260-df3c-4f62-b8f1-f4f5f305694b@66.81.247.155:443?path=%2F%3Fed%3D512&security=tls&encryption=none&insecure=0&host=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net&fp=chrome&type=ws&allowInsecure=0&sni=yyzsuabw9e3qd5ud7ihi5dxm96oglnsvr83cjojnm1efncfhr9ucordq.zjde5.de5.net#%F0%9F%8C%8E%20%40vmesspv
+vless://83f03646-fb28-44cc-9d2c-8853f6c09285@104.17.162.123:8443?path=%2F%3Fed%3D%23TELEGRAM-Yam%3Fed%3D512&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=r4fnviw9jl4i4rx.zjde5.de5.net&fp=random&type=ws&allowInsecure=0&sni=r4fnviw9jl4i4rx.zjde5.de5.net#@chthxyz - 61
+vless://3a4ddfac-e7da-48c9-9648-4a366109fc3a@api.steamsale.ir:443?path=%2FX0PX5Vup1qlVVzhxp6ic50a&security=tls&alpn=http%2F1.1&encryption=none&insecure=0&host=mtn.vpsmee.ir&fp=chrome&type=ws&allowInsecure=0&sni=cdn.vpsmee.ir#@chthxyz - 28

internal/config/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	InputPath   string `envconfig:"INPUT_PATH" default:"proxies.txt"`
 	OutputPath  string `envconfig:"OUTPUT_PATH" default:"valid.jsonl"`
 	GeoIPPath   string `envconfig:"GEOIP_PATH" default:"GeoLite2-Country.mmdb"`
+	TxtOutputPath string `envconfig:"TXT_OUTPUT_PATH" default:"valid.txt"`
 }
 
 // Load reads .env and processes environment variables

internal/sink/writer.go

@@ -35,4 +35,30 @@ func (w *JSONLWriter) Write(p *model.Proxy) error {
 
 func (w *JSONLWriter) Close() {
 	w.file.Close()
-}
+}
+
+
+type TextWriter struct {
+	file *os.File
+	mu   sync.Mutex
+}
+
+func NewText(path string) (*TextWriter, error) {
+	// We use O_APPEND | O_CREATE | O_WRONLY
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+	return &TextWriter{file: f}, nil
+}
+
+func (w *TextWriter) Write(p *model.Proxy) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	
+	// Just write the RawLink + NewLine
+	_, err := w.file.WriteString(p.RawLink + "\n")
+	return err
+}
+
+func (w *TextWriter) Close() { w.file.Close() }

internal/source/loader.go

@@ -2,60 +2,122 @@ package source
 
 import (
 	"bufio"
+	"encoding/base64"
+	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"strings"
+	"sync"
+	"time"
 )
 
-// Loader returns a channel of strings to keep memory usage low
-func LoadFromFile(path string) (<-chan string, error) {
-	file, err := os.Open(path)
-	if err != nil {
-		return nil, err
+// Load is the entry point. It determines if the input is a URL or a File.
+func Load(input string) (<-chan string, error) {
+	input = strings.TrimSpace(input)
+	out := make(chan string)
+
+	// If the input arg itself is a URL, fetch it directly
+	if strings.HasPrefix(input, "http://") || strings.HasPrefix(input, "https://") {
+		slog.Info("loading_from_remote_url", "url", input)
+		go func() {
+			defer close(out)
+			fetchAndStream(input, out)
+		}()
+		return out, nil
 	}
 
+	// Otherwise, treat it as a file (which might contain URLs!)
+	slog.Info("loading_from_file", "path", input)
+	return loadFromFileRecursive(input), nil
+}
+
+// loadFromFileRecursive reads a file line-by-line.
+// If a line is a URL, it fetches it. If it's a proxy, it sends it.
+func loadFromFileRecursive(path string) <-chan string {
 	out := make(chan string)
 
 	go func() {
-		defer file.Close()
 		defer close(out)
 
+		file, err := os.Open(path)
+		if err != nil {
+			slog.Error("file_open_failed", "path", path, "error", err)
+			return
+		}
+		defer file.Close()
+
+		var wg sync.WaitGroup
 		scanner := bufio.NewScanner(file)
-		// Increase buffer size for very long lines (some subscription links are huge)
-		buf := make([]byte, 0, 64*1024) 
+		buf := make([]byte, 0, 64*1024)
 		scanner.Buffer(buf, 1024*1024)
 
 		for scanner.Scan() {
 			line := strings.TrimSpace(scanner.Text())
-			if line != "" && !strings.HasPrefix(line, "#") {
+			if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") {
+				continue
+			}
+
+			// MAGIC: If the line inside the file is a URL, fetch it!
+			if strings.HasPrefix(line, "http://") || strings.HasPrefix(line, "https://") {
+				wg.Add(1)
+				// Fetch in parallel
+				go func(url string) {
+					defer wg.Done()
+					fetchAndStream(url, out)
+				}(line)
+			} else {
+				// It's just a raw proxy link
 				out <- line
 			}
 		}
+		
+		// Wait for all subscription fetches to finish before closing channel
+		wg.Wait()
 	}()
 
-	return out, nil
+	return out
 }
 
-// LoadFromURL streams directly from a URL (e.g., Github raw)
-func LoadFromURL(url string) (<-chan string, error) {
-	resp, err := http.Get(url)
+// fetchAndStream downloads content from a URL and parses it
+func fetchAndStream(url string, out chan<- string) {
+	client := &http.Client{Timeout: 15 * time.Second}
+	resp, err := client.Get(url)
 	if err != nil {
-		return nil, err
+		slog.Warn("subscription_fetch_failed", "url", url, "error", err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		slog.Warn("subscription_bad_status", "url", url, "status", resp.StatusCode)
+		return
 	}
 
-	out := make(chan string)
-	go func() {
-		defer resp.Body.Close()
-		defer close(out)
-		
-		scanner := bufio.NewScanner(resp.Body)
-		for scanner.Scan() {
-			line := strings.TrimSpace(scanner.Text())
-			if line != "" {
-				out <- line
-			}
-		}
-	}()
+	// Read full body to handle Base64 decoding if necessary
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	content := string(bodyBytes)
 
-	return out, nil
+	// STEP 1: Try Base64 Decode
+	// Many subscriptions return a base64 blob of proxies
+	if decoded, err := base64.StdEncoding.DecodeString(content); err == nil {
+		content = string(decoded)
+	} else if decoded, err := base64.RawStdEncoding.DecodeString(content); err == nil {
+		content = string(decoded)
+	}
+
+	// STEP 2: Split by Newline and Stream
+	lines := strings.Split(content, "\n")
+	count := 0
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line != "" && !strings.HasPrefix(line, "#") {
+			out <- line
+			count++
+		}
+	}
+	slog.Info("subscription_loaded", "url", url, "proxies_found", count)
 }