feat: add alive configs, store testing output

internal/config/config.go

@@ -24,6 +24,9 @@ type Config struct {
 	OutputPath  string `envconfig:"OUTPUT_PATH" default:"valid.jsonl"`
 	GeoIPPath   string `envconfig:"GEOIP_PATH" default:"GeoLite2-Country.mmdb"`
 	TxtOutputPath string `envconfig:"TXT_OUTPUT_PATH" default:"valid.txt"`
+	AliveOutputPath string `envconfig:"ALIVE_OUTPUT_PATH" default:"alive.jsonl"`
+	AliveTxtOutputPath string `envconfig:"ALIVE_TXT_OUTPUT_PATH" default:"alive.txt"`
+	DatasetOutputPath  string `envconfig:"DATASET_OUTPUT_PATH" default:"dataset.jsonl"`
 }
 
 // Load reads .env and processes environment variables

internal/dedup/filter.go

@@ -3,6 +3,7 @@ package dedup
 import (
 	"fmt"
 	"sync"
+	"find-me-internet/internal/model"
 )
 
 type Filter struct {
@@ -16,9 +17,11 @@ func New() *Filter {
 	}
 }
 
-// Check returns true if the item is NEW (not seen before)
-func (f *Filter) Seen(address string, port int) bool {
-	key := fmt.Sprintf("%s:%d", address, port)
+// Seen checks if the proxy is new.
+// Key format: "vless://1.2.3.4:443"
+// This allows the same IP to be scanned again if it uses a different protocol.
+func (f *Filter) Seen(p *model.Proxy) bool {
+	key := fmt.Sprintf("%s://%s:%d", p.Type, p.Address, p.Port)
 	
 	f.mu.RLock()
 	_, exists := f.seen[key]

internal/filter/network.go

@@ -2,7 +2,6 @@ package filter
 
 import (
 	"crypto/tls"
-	"log/slog"
 	"net"
 	"strconv"
 	"time"
@@ -18,53 +17,46 @@ func NewPipeline(timeout time.Duration) *Pipeline {
 	return &Pipeline{Timeout: timeout}
 }
 
+// Check performs cheap checks and updates the Proxy model with results.
+// Returns true ONLY if all checks pass.
 func (f *Pipeline) Check(p *model.Proxy) bool {
-	target := net.JoinHostPort(p.Address, strconv.Itoa(p.Port))
-	log := slog.With("target", target, "protocol", p.Type)
-
-	// 1. TCP Connectivity
-	start := time.Now()
+	// 1. TCP Check
 	if !f.checkTCP(p) {
-		log.Debug("tcp_connect_failed", "duration", time.Since(start))
 		p.IsOnline = false
+		p.Status = "dead"
+		p.FailureStage = "filter"
+		p.FailureReason = "tcp_timeout_or_refused"
 		return false
 	}
 	p.IsOnline = true
 
-	// 2. TLS Handshake
-	// Only proceed if protocol supports/requires TLS
+	// 2. TLS Check
+	// Determine if TLS is required
 	shouldCheckTLS := p.SNI != "" || p.Port == 443 || p.Type == model.TypeVLESS || p.Type == model.TypeTrojan
 	
 	if shouldCheckTLS {
 		sni := p.SNI
-		if sni == "" {
-			sni = p.Address // Fallback for handshake
-		}
+		if sni == "" { sni = p.Address }
 
-		startTLS := time.Now()
 		if !f.checkTLS(p, sni) {
-			log.Debug("tls_handshake_failed", 
-				"sni", sni, 
-				"duration", time.Since(startTLS),
-			)
 			p.IsTLSSecure = false
+			p.Status = "dead"
+			p.FailureStage = "filter"
+			p.FailureReason = "tls_handshake_failed"
 			return false
 		}
 		p.IsTLSSecure = true
-		log.Debug("network_checks_passed", "duration", time.Since(start))
-	} else {
-		log.Debug("network_checks_passed", "note", "tls_skipped_no_sni")
 	}
 
+	// If we got here, it passed the filter stage
+	p.FailureStage = "none" 
 	return true
 }
 
 func (f *Pipeline) checkTCP(p *model.Proxy) bool {
 	address := net.JoinHostPort(p.Address, strconv.Itoa(p.Port))
 	conn, err := net.DialTimeout("tcp", address, f.Timeout)
-	if err != nil {
-		return false
-	}
+	if err != nil { return false }
 	conn.Close()
 	return true
 }
@@ -72,16 +64,9 @@ func (f *Pipeline) checkTCP(p *model.Proxy) bool {
 func (f *Pipeline) checkTLS(p *model.Proxy, sni string) bool {
 	address := net.JoinHostPort(p.Address, strconv.Itoa(p.Port))
 	dialer := &net.Dialer{Timeout: f.Timeout}
-	
-	conf := &tls.Config{
-		InsecureSkipVerify: true,
-		ServerName:         sni,
-	}
-
+	conf := &tls.Config{InsecureSkipVerify: true, ServerName: sni}
 	conn, err := tls.DialWithDialer(dialer, "tcp", address, conf)
-	if err != nil {
-		return false
-	}
+	if err != nil { return false }
 	conn.Close()
 	return true
 }

internal/model/proxy.go

@@ -2,7 +2,6 @@ package model
 
 import "time"
 
-// ProxyType defines the protocol (vless, vmess, etc.)
 type ProxyType string
 
 const (
@@ -13,25 +12,25 @@ const (
 	TypeUnknown ProxyType = "unknown"
 )
 
-// Proxy represents a single internet access point
 type Proxy struct {
-	// Identity
+	// --- Identity ---
 	RawLink string    `json:"link"`
 	Type    ProxyType `json:"type"`
+	Address string    `json:"address"`
+	Port    int       `json:"port"`
+	Network string    `json:"network"`
+	SNI     string    `json:"sni"`
 
-	// Connection Details
-	Address string `json:"address"` // IP or Domain
-	Port    int    `json:"port"`
-	UUID    string `json:"uuid"`    // Or Password
-	SNI     string `json:"sni"`     // TLS Server Name Indicator
-	Network string `json:"network"` // tcp, ws, grpc, h2
+	// --- Enrichment ---
+	Country string    `json:"country"` // e.g., "US", "IR", "DE"
 
-	// Filter Stage Results
-	IsOnline    bool `json:"is_online"`     // TCP Connect success
-	IsTLSSecure bool `json:"is_tls_secure"` // TLS Handshake success
-
-	// Tester Stage Results
+	// --- Metrics ---
 	Latency    time.Duration `json:"latency_ms"`
-	Country    string        `json:"country_code"`
-	PacketLoss float64       `json:"packet_loss"` // 0.0 to 1.0
+	IsOnline   bool          `json:"is_online"`    // TCP Connect Status
+	IsTLSSecure bool         `json:"is_tls_secure"` // TLS Handshake Status
+
+	// --- Data Collection (The fields you want filled) ---
+	Status        string `json:"status"`         // "valid", "alive", "dead"
+	FailureStage  string `json:"failure_stage"`  // "filter", "tester", "none"
+	FailureReason string `json:"failure_reason"` // "tcp_timeout", "http_502", "tls_error", etc.
 }

internal/tester/runner.go

@@ -71,17 +71,20 @@ func (r *Runner) Test(p *model.Proxy) error {
 	}
 
 	// 5. HTTP Probe
-	startProbe := time.Now()
 	latency, err := r.measureLatency(port)
 	if err != nil {
-		log.Debug("http_probe_failed", 
-			"duration", time.Since(startProbe),
-			"error", err,
-		)
+		// SET THE MODEL VALUES HERE
+		p.Status = "alive" // It passed TCP, so it's "alive" but failed the test
+		p.FailureStage = "tester"
+		p.FailureReason = err.Error() // e.g., "http_timeout" or "status_502"
 		return err
 	}
 
+	// Success
 	p.Latency = latency
+	p.Status = "valid"
+	p.FailureStage = "none"
+	p.FailureReason = "none"
 	return nil
 }
 
@@ -95,15 +98,17 @@ func (r *Runner) measureLatency(port int) (time.Duration, error) {
 		Timeout: r.Timeout,
 	}
 
-	start := time.Now()
+	start := time.Now()	
 	resp, err := client.Get(r.TestURL)
-	if err != nil {
-		return 0, err
+	if err != nil {	
+		// Return specific error string for the model
+		return 0, fmt.Errorf("http_timeout_or_network_error")
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return 0, fmt.Errorf("unexpected_status_code_%d", resp.StatusCode)
+		// Return specific status code error
+		return 0, fmt.Errorf("http_error_%d", resp.StatusCode)
 	}
 
 	return time.Since(start), nil