feat: optimization

internal/config/config.go

@@ -10,27 +10,29 @@ import (
 
 type Config struct {
 	// App Settings
-	LogLevel string `envconfig:"LOG_LEVEL" default:"INFO"` // INFO, DEBUG, ERROR
-	Workers  int    `envconfig:"MAX_WORKERS" default:"10"`
+	LogLevel string `envconfig:"LOG_LEVEL" default:"INFO"`
+	Workers  int    `envconfig:"MAX_WORKERS" default:"20"`
 
-	// Paths
+	// Network Logic
+	TestURL     string        `envconfig:"TEST_URL" default:"http://cp.cloudflare.com"`
+	TcpTimeout  time.Duration `envconfig:"TCP_TIMEOUT" default:"2s"`
+	TestTimeout time.Duration `envconfig:"TEST_TIMEOUT" default:"10s"`
+
+	// File System Paths
 	SingBoxPath string `envconfig:"SING_BOX_PATH" default:"./bin/sing-box"`
-	
-	// Testing Parameters
-	TestURL      string        `envconfig:"TEST_URL" default:"http://cp.cloudflare.com"`
-	TcpTimeout   time.Duration `envconfig:"TCP_TIMEOUT" default:"2s"`
-	TestTimeout  time.Duration `envconfig:"TEST_TIMEOUT" default:"10s"`
+	InputPath   string `envconfig:"INPUT_PATH" default:"proxies.txt"`
+	OutputPath  string `envconfig:"OUTPUT_PATH" default:"valid.jsonl"`
+	GeoIPPath   string `envconfig:"GEOIP_PATH" default:"GeoLite2-Country.mmdb"`
 }
 
-// Load reads .env and maps variables to Config struct
+// Load reads .env and processes environment variables
 func Load() *Config {
-	// 1. Try loading .env file (optional, for local dev)
+	// Silently ignore if .env is missing (production might use real ENV vars)
 	_ = godotenv.Load()
 
 	var cfg Config
-	// 2. Process environment variables
 	if err := envconfig.Process("", &cfg); err != nil {
-		log.Fatalf("Failed to load config: %v", err)
+		log.Fatalf("Configuration Error: %v", err)
 	}
 	return &cfg
 }

internal/dedup/filter.go

@@ -0,0 +1,36 @@
+package dedup
+
+import (
+	"fmt"
+	"sync"
+)
+
+type Filter struct {
+	seen map[string]struct{}
+	mu   sync.RWMutex
+}
+
+func New() *Filter {
+	return &Filter{
+		seen: make(map[string]struct{}),
+	}
+}
+
+// Check returns true if the item is NEW (not seen before)
+func (f *Filter) Seen(address string, port int) bool {
+	key := fmt.Sprintf("%s:%d", address, port)
+	
+	f.mu.RLock()
+	_, exists := f.seen[key]
+	f.mu.RUnlock()
+
+	if exists {
+		return true
+	}
+
+	f.mu.Lock()
+	f.seen[key] = struct{}{}
+	f.mu.Unlock()
+	
+	return false
+}

internal/geoip/lookup.go

@@ -0,0 +1,40 @@
+package geoip
+
+import (
+	"net"
+	"github.com/oschwald/geoip2-golang"
+)
+
+type Database struct {
+	reader *geoip2.Reader
+}
+
+func Open(path string) (*Database, error) {
+	r, err := geoip2.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	return &Database{reader: r}, nil
+}
+
+func (d *Database) Lookup(ipStr string) string {
+	if d == nil || d.reader == nil {
+		return "N/A"
+	}
+	
+	ip := net.ParseIP(ipStr)
+	if ip == nil {
+		return "INVALID_IP"
+	}
+
+	record, err := d.reader.Country(ip)
+	if err != nil || record.Country.IsoCode == "" {
+		return "UNKNOWN"
+	}
+
+	return record.Country.IsoCode
+}
+
+func (d *Database) Close() {
+	d.reader.Close()
+}

internal/sink/writer.go

@@ -0,0 +1,38 @@
+package sink
+
+import (
+	"encoding/json"
+	"find-me-internet/internal/model"
+	"os"
+	"sync"
+)
+
+type JSONLWriter struct {
+	file *os.File
+	mu   sync.Mutex
+}
+
+func NewJSONL(path string) (*JSONLWriter, error) {
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return nil, err
+	}
+	return &JSONLWriter{file: f}, nil
+}
+
+func (w *JSONLWriter) Write(p *model.Proxy) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	data, err := json.Marshal(p)
+	if err != nil {
+		return err
+	}
+	
+	_, err = w.file.Write(append(data, '\n'))
+	return err
+}
+
+func (w *JSONLWriter) Close() {
+	w.file.Close()
+}

internal/source/loader.go

@@ -0,0 +1,61 @@
+package source
+
+import (
+	"bufio"
+	"net/http"
+	"os"
+	"strings"
+)
+
+// Loader returns a channel of strings to keep memory usage low
+func LoadFromFile(path string) (<-chan string, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan string)
+
+	go func() {
+		defer file.Close()
+		defer close(out)
+
+		scanner := bufio.NewScanner(file)
+		// Increase buffer size for very long lines (some subscription links are huge)
+		buf := make([]byte, 0, 64*1024) 
+		scanner.Buffer(buf, 1024*1024)
+
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if line != "" && !strings.HasPrefix(line, "#") {
+				out <- line
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+// LoadFromURL streams directly from a URL (e.g., Github raw)
+func LoadFromURL(url string) (<-chan string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	out := make(chan string)
+	go func() {
+		defer resp.Body.Close()
+		defer close(out)
+		
+		scanner := bufio.NewScanner(resp.Body)
+		for scanner.Scan() {
+			line := strings.TrimSpace(scanner.Text())
+			if line != "" {
+				out <- line
+			}
+		}
+	}()
+
+	return out, nil
+}