// 1. 判断是否需要进行泛解析处理, 需要的话就收集泛解析域名的 host
// 2. 调用 EnumerateSubdomainsWithCtx 去跑子域名
// 3. 使用协程处理子域名通道中的结果
// 3.1 判断该子域名是否是属于主域名
// 3.2 替换掉 *.
// 3.3 使用 filterAndMatchSubdomain 匹配用户自定义的正则表达式
// 3.4 使用 map 做去重 , 并且使用 map 记录去重后每个数据源的结果数量
// 3.5 将结果转换为 resolve.HostEntry, 如果需要去除泛解析的话就添加到泛解析的任务队列中
// 3.6 等待协程退出 => 处理结果的协程
// 3.7 输出 统计操作
func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string, writers []io.Writer) error {
gologger.Info().Msgf("Enumerating subdomains for %s\n", domain)
// Check if the user has asked to remove wildcards explicitly.
// If yes, create the resolution pool and get the wildcards for the current domain
var resolutionPool *resolve.ResolutionPool
if r.options.RemoveWildcard {
resolutionPool = r.resolverClient.NewResolutionPool(r.options.Threads, r.options.RemoveWildcard)
err := resolutionPool.InitWildcards(domain)
if err != nil {
// Log the error but don't quit.
gologger.Warning().Msgf("Could not get wildcards for domain %s: %s\n", domain, err)
}
}
// Run the passive subdomain enumeration
now := time.Now()
passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute, passive.WithCustomRateLimit(r.rateLimit))
wg := &sync.WaitGroup{}
wg.Add(1)
// Create a unique map for filtering duplicate subdomains out
uniqueMap := make(map[string]resolve.HostEntry)
// Create a map to track sources for each host
sourceMap := make(map[string]map[string]struct{})
skippedCounts := make(map[string]int)
// Process the results in a separate goroutine
go func() {
for result := range passiveResults {
switch result.Type {
case subscraping.Error:
gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error)
case subscraping.Subdomain:
// Validate the subdomain found and remove wildcards from
// 判断是否为主域名的子域
if !strings.HasSuffix(result.Value, "."+domain) {
skippedCounts[result.Source]++
continue
}
subdomain := strings.ReplaceAll(strings.ToLower(result.Value), "*.", "")
// 正则匹配 提取出来子域名
if matchSubdomain := r.filterAndMatchSubdomain(subdomain); matchSubdomain {
if _, ok := uniqueMap[subdomain]; !ok {
sourceMap[subdomain] = make(map[string]struct{})
}
// Log the verbose message about the found subdomain per source
if _, ok := sourceMap[subdomain][result.Source]; !ok {
gologger.Verbose().Label(result.Source).Msg(subdomain)
}
sourceMap[subdomain][result.Source] = struct{}{}
// Check if the subdomain is a duplicate. If not,
// send the subdomain for resolution.
if _, ok := uniqueMap[subdomain]; ok {
skippedCounts[result.Source]++
continue
}
hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source}
uniqueMap[subdomain] = hostEntry
// If the user asked to remove wildcard then send on the resolve
// queue. Otherwise, if mode is not verbose print the results on
// the screen as they are discovered.
if r.options.RemoveWildcard {
resolutionPool.Tasks <- hostEntry
}
}
}
}
// Close the task channel only if wildcards are asked to be removed
if r.options.RemoveWildcard {
close(resolutionPool.Tasks)
}
wg.Done()
}()
// If the user asked to remove wildcards, listen from the results
// queue and write to the map. At the end, print the found results to the screen
foundResults := make(map[string]resolve.Result)
if r.options.RemoveWildcard {
// Process the results coming from the resolutions pool
for result := range resolutionPool.Results {
switch result.Type {
case resolve.Error:
gologger.Warning().Msgf("Could not resolve host: %s\n", result.Error)
case resolve.Subdomain:
// Add the found subdomain to a map.
if _, ok := foundResults[result.Host]; !ok {
foundResults[result.Host] = result
}
}
}
}
wg.Wait()
outputWriter := NewOutputWriter(r.options.JSON)
// Now output all results in output writers
// 遍历好所有的 writer 进行输出操作
var err error
for _, writer := range writers {
if r.options.HostIP {
err = outputWriter.WriteHostIP(domain, foundResults, writer)
} else {
if r.options.RemoveWildcard {
err = outputWriter.WriteHostNoWildcard(domain, foundResults, writer)
} else {
if r.options.CaptureSources {
err = outputWriter.WriteSourceHost(domain, sourceMap, writer)
} else {
err = outputWriter.WriteHost(domain, uniqueMap, writer)
}
}
}
if err != nil {
gologger.Error().Msgf("Could not write results for %s: %s\n", domain, err)
return err
}
}
// Show found subdomain count in any case.
duration := durafmt.Parse(time.Since(now)).LimitFirstN(maxNumCount).String()
var numberOfSubDomains int
if r.options.RemoveWildcard {
numberOfSubDomains = len(foundResults)
} else {
numberOfSubDomains = len(uniqueMap)
}
if r.options.ResultCallback != nil {
if r.options.RemoveWildcard {
for host, result := range foundResults {
r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source})
}
} else {
for _, v := range uniqueMap {
r.options.ResultCallback(&v)
}
}
}
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration)
if r.options.Statistics {
gologger.Info().Msgf("Printing source statistics for %s", domain)
statistics := r.passiveAgent.GetStatistics()
// This is a hack to remove the skipped count from the statistics
// as we don't want to show it in the statistics.
// TODO: Design a better way to do this.
for source, count := range skippedCounts {
if stat, ok := statistics[source]; ok {
stat.Results -= count
statistics[source] = stat
}
}
printStatistics(statistics)
}
return nil
}
