feat(phase-2.2): synthesis activities (5 activities)

Activities for the synthesis workflow pipeline: 1. ChunkAndEmbedActivity — deterministic chunk ID + ingest via memory service 2. ExtractEntitiesActivity — wiki-link, proper noun, technical term extraction 3. ExtractFactsActivity — verb pattern matching (uses/runs/has/is/depends_on) 4. DetectContradictionsActivity — query existing facts + pre-filter contradictions 5. PersistSynthesisActivity — save entities + facts to memory service Entity extraction patterns: - [[WikiLinks]] → 0.95 confidence - ProperNouns → 0.70 confidence - TECHNICAL_TERMS/camelCase → 0.65 confidence - Deduplication across patterns Fact extraction: - 6 verb patterns (uses, runs_on, has, is, depends_on, connects_to) - Confidence boost when subject/object are known entities Contradiction detection: - Query memory service for existing facts about same subject - Pre-filter: subject match + different object - Severity: low/medium/high based on similarity score - Auto-resolve low severity, queue review for medium/high Tests: 10 pass (wiki links, proper nouns, tech terms, dedup, verb patterns, empty text, contradicts, contains, common, classify) Build: clean, 35 packages pass
2026-09-08 17:38:48 -07:00
parent 7be93d1d16
commit 5342de37cc
2 changed files with 484 additions and 0 deletions
@@ -0,0 +1,354 @@
+package activity
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"log/slog"
+	"regexp"
+	"strings"
+
+	"github.com/rockliang/poimen/workflows/internal/memory"
+)
+
+// SynthesisActivities holds dependencies for synthesis pipeline activities.
+type SynthesisActivities struct {
+	memClient *memory.Client
+}
+
+// NewSynthesisActivities creates synthesis activities with a memory service client.
+func NewSynthesisActivities(memClient *memory.Client) *SynthesisActivities {
+	return &SynthesisActivities{memClient: memClient}
+}
+
+// SynthesisInput mirrors workflow.SynthesisInput for activity deserialization.
+type SynthesisInput struct {
+	Project string   `json:"project"`
+	Source  string   `json:"source"`
+	Text    string   `json:"text"`
+	Kind    string   `json:"kind"`
+	Tags    []string `json:"tags,omitempty"`
+}
+
+// ExtractedEntity represents an entity found during synthesis.
+type ExtractedEntity struct {
+	Name       string  `json:"name"`
+	EntityType string  `json:"entity_type"`
+	Confidence float64 `json:"confidence"`
+}
+
+// ExtractedFact represents a fact extracted during synthesis.
+type ExtractedFact struct {
+	Subject    string  `json:"subject"`
+	Predicate  string  `json:"predicate"`
+	Object     string  `json:"object"`
+	Confidence float64 `json:"confidence"`
+}
+
+// ContradictionResult represents a contradiction detection result.
+type ContradictionResult struct {
+	FactA        ExtractedFact `json:"fact_a"`
+	FactB        ExtractedFact `json:"fact_b"`
+	Severity     string        `json:"severity"`
+	AutoResolved bool          `json:"auto_resolved"`
+	QueuedReview bool          `json:"queued_review"`
+}
+
+// PersistInput groups all synthesis results for persistence.
+type PersistInput struct {
+	ChunkID        string                `json:"chunk_id"`
+	Project        string                `json:"project"`
+	Source         string                `json:"source"`
+	Kind           string                `json:"kind"`
+	Entities       []ExtractedEntity     `json:"entities"`
+	Facts          []ExtractedFact       `json:"facts"`
+	Contradictions []ContradictionResult `json:"contradictions"`
+}
+
+// ChunkAndEmbedActivity chunks text and generates a chunk ID.
+// Stage 1: Creates a deterministic chunk ID from content hash,
+// then ingests via memory service for embedding generation.
+func (s *SynthesisActivities) ChunkAndEmbedActivity(ctx context.Context, input SynthesisInput) (string, error) {
+	logger := slog.Default()
+
+	// Generate deterministic chunk ID from content
+	hash := sha256.Sum256([]byte(input.Text))
+	chunkID := "chunk-" + hex.EncodeToString(hash[:8])
+
+	logger.Info("chunking text", "chunk_id", chunkID, "text_len", len(input.Text))
+
+	// Ingest via memory service (generates embedding)
+	_, err := s.memClient.Ingest(ctx, &memory.IngestRequest{
+		Project: input.Project,
+		Source:  input.Source,
+		Kind:    input.Kind,
+		Text:    input.Text,
+		Metadata: map[string]interface{}{
+			"chunk_id": chunkID,
+			"tags":     input.Tags,
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("ingest chunk: %w", err)
+	}
+
+	return chunkID, nil
+}
+
+// ExtractEntitiesActivity extracts entities from text using pattern matching
+// and wiki-link detection. LLM extraction is a future enhancement.
+// Stage 2: Returns entities with confidence scores.
+func (s *SynthesisActivities) ExtractEntitiesActivity(ctx context.Context, chunkID string, text string) ([]ExtractedEntity, error) {
+	logger := slog.Default()
+	logger.Info("extracting entities", "chunk_id", chunkID)
+
+	entities := make([]ExtractedEntity, 0)
+	seen := make(map[string]bool)
+
+	// Pattern 1: Wiki-link extraction [[EntityName]]
+	wikiPattern := regexp.MustCompile(`\[\[([^\]]+)\]\]`)
+	for _, match := range wikiPattern.FindAllStringSubmatch(text, -1) {
+		name := strings.TrimSpace(match[1])
+		if !seen[name] {
+			entities = append(entities, ExtractedEntity{
+				Name:       name,
+				EntityType: "reference",
+				Confidence: 0.95,
+			})
+			seen[name] = true
+		}
+	}
+
+	// Pattern 2: Capitalized proper nouns (simple NER)
+	properNounPattern := regexp.MustCompile(`\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b`)
+	for _, match := range properNounPattern.FindAllStringSubmatch(text, -1) {
+		name := match[1]
+		if !seen[name] && !isCommonWord(name) && len(name) > 2 {
+			entities = append(entities, ExtractedEntity{
+				Name:       name,
+				EntityType: classifyEntity(name),
+				Confidence: 0.70,
+			})
+			seen[name] = true
+		}
+	}
+
+	// Pattern 3: Technical terms (ALL_CAPS or camelCase)
+	techPattern := regexp.MustCompile(`\b([A-Z][A-Z_]{2,}|[a-z]+[A-Z][a-zA-Z]+)\b`)
+	for _, match := range techPattern.FindAllStringSubmatch(text, -1) {
+		name := match[1]
+		if !seen[name] {
+			entities = append(entities, ExtractedEntity{
+				Name:       name,
+				EntityType: "technical",
+				Confidence: 0.65,
+			})
+			seen[name] = true
+		}
+	}
+
+	logger.Info("entities extracted", "count", len(entities))
+	return entities, nil
+}
+
+// ExtractFactsActivity extracts subject-predicate-object facts from text.
+// Stage 3: Pattern-based extraction with entity context.
+func (s *SynthesisActivities) ExtractFactsActivity(ctx context.Context, chunkID string, text string, entities []ExtractedEntity) ([]ExtractedFact, error) {
+	logger := slog.Default()
+	logger.Info("extracting facts", "chunk_id", chunkID, "entity_count", len(entities))
+
+	facts := make([]ExtractedFact, 0)
+
+	// Build entity name set for matching
+	entityNames := make(map[string]bool)
+	for _, e := range entities {
+		entityNames[strings.ToLower(e.Name)] = true
+	}
+
+	// Pattern: "X uses/runs/has Y"
+	verbPatterns := []struct {
+		pattern   *regexp.Regexp
+		predicate string
+	}{
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+uses?\s+(.+?)(?:\.|,|$)`), "uses"},
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+runs?\s+(?:on\s+)?(.+?)(?:\.|,|$)`), "runs_on"},
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+(?:has|have)\s+(.+?)(?:\.|,|$)`), "has"},
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+(?:is|are)\s+(.+?)(?:\.|,|$)`), "is"},
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+(?:depends?\s+on|requires?)\s+(.+?)(?:\.|,|$)`), "depends_on"},
+		{regexp.MustCompile(`(?i)(\w+(?:\s+\w+)?)\s+(?:connects?\s+to|talks?\s+to)\s+(.+?)(?:\.|,|$)`), "connects_to"},
+	}
+
+	for _, vp := range verbPatterns {
+		for _, match := range vp.pattern.FindAllStringSubmatch(text, -1) {
+			subject := strings.TrimSpace(match[1])
+			object := strings.TrimSpace(match[2])
+
+			// Boost confidence if subject/object are known entities
+			confidence := 0.60
+			if entityNames[strings.ToLower(subject)] {
+				confidence += 0.15
+			}
+			if entityNames[strings.ToLower(object)] {
+				confidence += 0.15
+			}
+
+			facts = append(facts, ExtractedFact{
+				Subject:    subject,
+				Predicate:  vp.predicate,
+				Object:     object,
+				Confidence: confidence,
+			})
+		}
+	}
+
+	logger.Info("facts extracted", "count", len(facts))
+	return facts, nil
+}
+
+// DetectContradictionsActivity detects contradictions between new facts
+// and existing knowledge. Uses pre-filter to avoid unnecessary comparisons.
+// Stage 4: Returns contradictions with severity and review status.
+func (s *SynthesisActivities) DetectContradictionsActivity(ctx context.Context, project string, facts []ExtractedFact) ([]ContradictionResult, error) {
+	logger := slog.Default()
+	logger.Info("detecting contradictions", "project", project, "fact_count", len(facts))
+
+	contradictions := make([]ContradictionResult, 0)
+
+	for _, fact := range facts {
+		// Query existing facts about the same subject
+		query := fmt.Sprintf("%s %s", fact.Subject, fact.Predicate)
+		results, err := s.memClient.Query(ctx, &memory.QueryRequest{
+			Project:     project,
+			Query:       query,
+			LevelFilter: []string{"L1", "L2"},
+			Floor:       0.7,
+			Limit:       5,
+		})
+		if err != nil {
+			logger.Warn("query for contradictions failed", "error", err, "subject", fact.Subject)
+			continue // Non-fatal: skip this fact
+		}
+
+		for _, r := range results.Results {
+			// Pre-filter: check if result mentions same subject + different object
+			if containsSubject(r.Text, fact.Subject) && contradicts(r.Text, fact) {
+				severity := "low"
+				if r.Score > 0.9 {
+					severity = "high"
+				} else if r.Score > 0.8 {
+					severity = "medium"
+				}
+
+				autoResolved := severity == "low"
+				contradictions = append(contradictions, ContradictionResult{
+					FactA: ExtractedFact{
+						Subject:   fact.Subject,
+						Predicate: fact.Predicate,
+						Object:    r.Text,
+					},
+					FactB:        fact,
+					Severity:     severity,
+					AutoResolved: autoResolved,
+					QueuedReview: !autoResolved,
+				})
+			}
+		}
+	}
+
+	logger.Info("contradictions detected", "count", len(contradictions))
+	return contradictions, nil
+}
+
+// PersistSynthesisActivity saves all synthesis results to the memory service.
+// Stage 5: Persists entities, facts, and queues contradictions for review.
+func (s *SynthesisActivities) PersistSynthesisActivity(ctx context.Context, input PersistInput) error {
+	logger := slog.Default()
+	logger.Info("persisting synthesis results",
+		"chunk_id", input.ChunkID,
+		"entities", len(input.Entities),
+		"facts", len(input.Facts),
+		"contradictions", len(input.Contradictions),
+	)
+
+	// Persist entities as knowledge records
+	for _, entity := range input.Entities {
+		_, err := s.memClient.Ingest(ctx, &memory.IngestRequest{
+			Project: input.Project,
+			Source:  input.Source,
+			Kind:    "L1",
+			Text:    fmt.Sprintf("Entity: %s (type: %s, confidence: %.2f)", entity.Name, entity.EntityType, entity.Confidence),
+			Metadata: map[string]interface{}{
+				"chunk_id":    input.ChunkID,
+				"entity_type": entity.EntityType,
+				"entity_name": entity.Name,
+			},
+		})
+		if err != nil {
+			logger.Warn("failed to persist entity", "entity", entity.Name, "error", err)
+		}
+	}
+
+	// Persist facts
+	for _, fact := range input.Facts {
+		_, err := s.memClient.Ingest(ctx, &memory.IngestRequest{
+			Project: input.Project,
+			Source:  input.Source,
+			Kind:    "L1",
+			Text:    fmt.Sprintf("%s %s %s", fact.Subject, fact.Predicate, fact.Object),
+			Metadata: map[string]interface{}{
+				"chunk_id":  input.ChunkID,
+				"subject":   fact.Subject,
+				"predicate": fact.Predicate,
+				"object":    fact.Object,
+			},
+		})
+		if err != nil {
+			logger.Warn("failed to persist fact", "subject", fact.Subject, "error", err)
+		}
+	}
+
+	logger.Info("synthesis persisted", "chunk_id", input.ChunkID)
+	return nil
+}
+
+// --- helpers ---
+
+func isCommonWord(word string) bool {
+	common := map[string]bool{
+		"The": true, "This": true, "That": true, "These": true,
+		"There": true, "When": true, "Where": true, "What": true,
+		"Which": true, "How": true, "But": true, "And": true,
+		"For": true, "Not": true, "You": true, "All": true,
+		"Can": true, "Her": true, "Was": true, "One": true,
+		"Our": true, "Out": true, "Are": true, "Has": true,
+		"Its": true, "May": true, "New": true, "Now": true,
+		"Old": true, "See": true, "Way": true, "Who": true,
+	}
+	return common[word]
+}
+
+func classifyEntity(name string) string {
+	toolPatterns := []string{"Kubernetes", "Docker", "Nginx", "Redis", "Postgres", "ArgoCD", "Terraform", "Helm"}
+	for _, t := range toolPatterns {
+		if strings.EqualFold(name, t) {
+			return "tool"
+		}
+	}
+	return "concept"
+}
+
+func containsSubject(text, subject string) bool {
+	return strings.Contains(strings.ToLower(text), strings.ToLower(subject))
+}
+
+func contradicts(existingText string, newFact ExtractedFact) bool {
+	// Simple heuristic: if existing text mentions subject with a different value
+	// for the same predicate pattern, it might contradict
+	lower := strings.ToLower(existingText)
+	subjectLower := strings.ToLower(newFact.Subject)
+	objectLower := strings.ToLower(newFact.Object)
+
+	// If text mentions subject but NOT the same object, potential contradiction
+	return strings.Contains(lower, subjectLower) && !strings.Contains(lower, objectLower)
+}