diff --git a/common/httpx/title.go b/common/httpx/title.go index 084d16cad..dd784f12e 100644 --- a/common/httpx/title.go +++ b/common/httpx/title.go @@ -16,6 +16,7 @@ var ( cutset = "\n\t\v\f\r" reTitle = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`) reContentType = regexp.MustCompile(`(?is)]*?charset\s*=\s*["']?([^"'\s;>/]+)`) + reGenerator = regexp.MustCompile(`(?i)]*?name\s*=\s*["']generator["'][^>]*?content\s*=\s*["']([^"']+)["']|]*?content\s*=\s*["']([^"']+)["'][^>]*?name\s*=\s*["']generator["']`) supportedTitleMimeTypes = []string{ "text/html", "application/xhtml+xml", @@ -50,6 +51,25 @@ func ExtractTitle(r *Response) (title string) { return title } +// ExtractGenerator extracts the content of from a response. +func ExtractGenerator(r *Response) string { + matches := reGenerator.FindSubmatch(r.Data) + var raw []byte + if len(matches) >= 3 { + if len(matches[1]) > 0 { + raw = matches[1] + } else if len(matches[2]) > 0 { + raw = matches[2] + } + } + if len(raw) == 0 { + return "" + } + // Postgres rejects NUL in TEXT, and one bad value rolls back the whole batch. + value := strings.ReplaceAll(string(raw), "\x00", "") + return strings.TrimSpace(html.UnescapeString(value)) +} + func CanHaveTitleTag(mimeType string) bool { return slices.Contains(supportedTitleMimeTypes, strings.ToLower(strings.TrimSpace(mimeType))) } diff --git a/common/httpx/title_test.go b/common/httpx/title_test.go index 97f68cbcb..840435297 100644 --- a/common/httpx/title_test.go +++ b/common/httpx/title_test.go @@ -98,6 +98,25 @@ func TestExtractTitleDecodesBig5HeaderCharset(t *testing.T) { require.Equal(t, "中文", ExtractTitle(resp)) } +func TestExtractGenerator(t *testing.T) { + tests := []struct { + name string + html string + want string + }{ + {"name then content", ``, "WordPress 6.4"}, + {"content then name", ``, "Drupal 10"}, + {"decoded entity", ``, "Bob's CMS"}, + {"nul stripped", "", "BadCMS"}, + {"absent", ``, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, ExtractGenerator(&Response{Data: []byte(tt.html)})) + }) + } +} + func TestDecodeDataBig5HeaderCharsetIsExact(t *testing.T) { big5 := []byte{0xa4, 0xa4, 0xa4, 0xe5} decoded, err := DecodeData(big5, http.Header{"Content-Type": []string{`text/html; charset="Big5"`}}) diff --git a/internal/db/mysql.go b/internal/db/mysql.go index 59432fab6..a4c7c8b1c 100644 --- a/internal/db/mysql.go +++ b/internal/db/mysql.go @@ -71,6 +71,7 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { content_length INT, content_type VARCHAR(255), title TEXT, + generator TEXT, webserver VARCHAR(255), response_time VARCHAR(50), location TEXT, @@ -174,6 +175,9 @@ func (m *mysqlDatabase) EnsureSchema(ctx context.Context) error { if err := m.ensureColumn(ctx, "cpe", "JSON"); err != nil { return fmt.Errorf("failed to ensure cpe column: %w", err) } + if err := m.ensureColumn(ctx, "generator", "TEXT"); err != nil { + return fmt.Errorf("failed to ensure generator column: %w", err) + } return nil } @@ -216,7 +220,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result query := fmt.Sprintf(` INSERT INTO %s ( timestamp, url, input, host, port, scheme, path, method, final_url, - status_code, content_length, content_type, title, webserver, response_time, + status_code, content_length, content_type, title, generator, webserver, response_time, location, body, body_preview, raw_header, request, host_ip, a, aaaa, cname, resolvers, body_fqdn, body_domains, sni, tech, hash, favicon, favicon_md5, favicon_path, favicon_url, jarm_hash, @@ -229,7 +233,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result cpe ) VALUES ( ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, @@ -274,7 +278,7 @@ func (m *mysqlDatabase) InsertBatch(ctx context.Context, results []runner.Result _, err = stmt.ExecContext(ctx, r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, - r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.WebServer, r.ResponseTime, + r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.Generator, r.WebServer, r.ResponseTime, r.Location, r.ResponseBody, r.BodyPreview, r.RawHeaders, r.Request, r.HostIP, aJSON, aaaaJSON, cnameJSON, resolversJSON, fqdnJSON, domainsJSON, r.SNI, techJSON, hashJSON, r.FavIconMMH3, r.FavIconMD5, r.FaviconPath, r.FaviconURL, r.JarmHash, diff --git a/internal/db/postgres.go b/internal/db/postgres.go index 24cbb70bf..268cdb8f5 100644 --- a/internal/db/postgres.go +++ b/internal/db/postgres.go @@ -72,6 +72,7 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { content_length INTEGER, content_type TEXT, title TEXT, + generator TEXT, webserver TEXT, response_time TEXT, location TEXT, @@ -163,6 +164,7 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { -- migration framework (e.g. golang-migrate / goose) once more schema -- changes accumulate. ALTER TABLE %s ADD COLUMN IF NOT EXISTS cpe JSONB; + ALTER TABLE %s ADD COLUMN IF NOT EXISTS generator TEXT; CREATE INDEX IF NOT EXISTS %s ON %s(timestamp DESC); CREATE INDEX IF NOT EXISTS %s ON %s(url); @@ -170,6 +172,7 @@ func (p *postgresDatabase) EnsureSchema(ctx context.Context) error { CREATE INDEX IF NOT EXISTS %s ON %s(status_code); CREATE INDEX IF NOT EXISTS %s ON %s USING GIN(tech); `, + tableName, tableName, tableName, idxTimestamp, tableName, @@ -204,7 +207,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res query := fmt.Sprintf(` INSERT INTO %s ( timestamp, url, input, host, port, scheme, path, method, final_url, - status_code, content_length, content_type, title, webserver, response_time, + status_code, content_length, content_type, title, generator, webserver, response_time, location, body, body_preview, raw_header, request, host_ip, a, aaaa, cname, resolvers, body_fqdn, body_domains, sni, tech, hash, favicon, favicon_md5, favicon_path, favicon_url, jarm_hash, @@ -217,17 +220,17 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res cpe ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, - $10, $11, $12, $13, $14, $15, - $16, $17, $18, $19, $20, - $21, $22, $23, $24, $25, $26, $27, $28, - $29, $30, $31, $32, $33, $34, $35, - $36, $37, $38, $39, $40, $41, - $42, $43, $44, $45, $46, $47, - $48, $49, $50, $51, $52, - $53, $54, - $55, $56, $57, $58, $59, - $60, $61, $62, - $63 + $10, $11, $12, $13, $14, $15, $16, + $17, $18, $19, $20, $21, + $22, $23, $24, $25, $26, $27, $28, $29, + $30, $31, $32, $33, $34, $35, $36, + $37, $38, $39, $40, $41, $42, + $43, $44, $45, $46, $47, $48, + $49, $50, $51, $52, $53, + $54, $55, + $56, $57, $58, $59, $60, + $61, $62, $63, + $64 )`, tableName) stmt, err := tx.PrepareContext(ctx, query) @@ -253,7 +256,7 @@ func (p *postgresDatabase) InsertBatch(ctx context.Context, results []runner.Res _, err = stmt.ExecContext(ctx, r.Timestamp, r.URL, r.Input, r.Host, r.Port, r.Scheme, r.Path, r.Method, r.FinalURL, - r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.WebServer, r.ResponseTime, + r.StatusCode, r.ContentLength, r.ContentType, r.Title, r.Generator, r.WebServer, r.ResponseTime, r.Location, r.ResponseBody, r.BodyPreview, r.RawHeaders, r.Request, r.HostIP, pq.Array(r.A), pq.Array(r.AAAA), pq.Array(r.CNAMEs), pq.Array(r.Resolvers), pq.Array(r.Fqdns), pq.Array(r.Domains), r.SNI, pq.Array(r.Technologies), hashJSON, r.FavIconMMH3, r.FavIconMD5, r.FaviconPath, r.FaviconURL, r.JarmHash, diff --git a/runner/runner.go b/runner/runner.go index d20b05242..55663382c 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -2197,8 +2197,10 @@ retry: } var title string + var generator string if httpx.CanHaveTitleTag(resp.GetHeaderPart("Content-Type", ";")) { title = httpx.ExtractTitle(resp) + generator = httpx.ExtractGenerator(resp) } if scanopts.OutputTitle && title != "" { @@ -2754,6 +2756,7 @@ retry: Location: resp.GetHeaderPart("Location", ";"), ContentType: resp.GetHeaderPart("Content-Type", ";"), Title: title, + Generator: generator, str: builder.String(), VHost: isvhost, WebServer: serverHeader, diff --git a/runner/types.go b/runner/types.go index 1eaa3608e..e86879ea7 100644 --- a/runner/types.go +++ b/runner/types.go @@ -50,6 +50,7 @@ type Result struct { Input string `json:"input,omitempty" csv:"input" md:"input" mapstructure:"input"` Location string `json:"location,omitempty" csv:"location" md:"location" mapstructure:"location"` Title string `json:"title,omitempty" csv:"title" md:"title" mapstructure:"title"` + Generator string `json:"generator,omitempty" csv:"generator" md:"generator" mapstructure:"generator"` str string `json:"-" csv:"-" md:"-" mapstructure:"-"` Scheme string `json:"scheme,omitempty" csv:"scheme" md:"scheme" mapstructure:"scheme"` Error string `json:"error,omitempty" csv:"error" md:"error" mapstructure:"error"`