Add complete MIDI search and synthesis pipeline

Backend (MVP):
- Express API with /search and /fetch endpoints
- BitMidi and Dongrays search adapters with scoring heuristics
- CORS proxy with disk caching and validation
- Quality assessment and file size limits

Frontend Integration:
- Switch to @tonejs/midi parser for reliable MIDI parsing
- MIDIService for backend API communication
- MotifEngine updated to try real MIDI, fallback to synthetic
- Enhanced error handling and logging

Development:
- Concurrent frontend/backend npm scripts
- Basic project documentation

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

server/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "motif-backend",
+  "version": "0.1.0",
+  "description": "MIDI search and proxy backend for Motif",
+  "type": "module",
+  "main": "dist/server.js",
+  "scripts": {
+    "dev": "tsx watch src/server.ts",
+    "build": "tsc",
+    "start": "node dist/server.js"
+  },
+  "dependencies": {
+    "express": "^4.18.2",
+    "cors": "^2.8.5",
+    "cheerio": "^1.0.0-rc.12",
+    "crypto": "^1.0.1"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.21",
+    "@types/cors": "^2.8.17",
+    "@types/node": "^20.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.0.0"
+  }
+}

server/src/adapters/BitMidiAdapter.ts

@@ -0,0 +1,75 @@
+import type { SearchAdapter, MIDICandidate } from '../types.js';
+import { ScoreUtils } from '../utils/ScoreUtils.js';
+
+export class BitMidiAdapter implements SearchAdapter {
+  name = 'bitmidi';
+  private baseUrl = 'https://bitmidi.com';
+
+  async search(query: string): Promise<MIDICandidate[]> {
+    try {
+      const searchUrl = `${this.baseUrl}/search?q=${encodeURIComponent(query)}`;
+      const response = await fetch(searchUrl, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; MotifBot/1.0)'
+        }
+      });
+
+      if (!response.ok) {
+        throw new Error(`BitMidi search failed: ${response.status}`);
+      }
+
+      const html = await response.text();
+      return this.parseSearchResults(html, query);
+    } catch (error) {
+      console.error('BitMidi search error:', error);
+      return [];
+    }
+  }
+
+  private parseSearchResults(html: string, query: string): MIDICandidate[] {
+    const candidates: MIDICandidate[] = [];
+    
+    // Simple regex-based parsing for MVP (would use cheerio in production)
+    const linkRegex = /<a[^>]*href="([^"]*)"[^>]*>([^<]*)</gi;
+    let match;
+
+    while ((match = linkRegex.exec(html)) !== null && candidates.length < 10) {
+      const [, href, title] = match;
+      
+      // Look for MIDI file links
+      if (href.includes('/midi/') && !href.includes('.mp3') && !href.includes('.wav')) {
+        const fullUrl = href.startsWith('http') ? href : `${this.baseUrl}${href}`;
+        const midiUrl = this.extractMidiUrl(fullUrl);
+        
+        if (midiUrl && title.trim()) {
+          const confidence = ScoreUtils.calculateConfidence(title, query, this.name);
+          
+          candidates.push({
+            id: `bitmidi_${Buffer.from(fullUrl).toString('base64').slice(0, 16)}`,
+            title: title.trim(),
+            source: 'bitmidi',
+            pageUrl: fullUrl,
+            midiUrl: midiUrl,
+            confidence
+          });
+        }
+      }
+    }
+
+    return candidates
+      .filter(c => c.confidence > 0.3) // Filter low confidence matches
+      .sort((a, b) => b.confidence - a.confidence)
+      .slice(0, 5); // Top 5 results
+  }
+
+  private extractMidiUrl(pageUrl: string): string {
+    // For BitMidi, the MIDI download is typically at the same path with .mid extension
+    // or through a download endpoint
+    if (pageUrl.includes('/midi/')) {
+      // Try direct .mid file first
+      const basePath = pageUrl.replace(/\/$/, '');
+      return `${basePath}.mid`;
+    }
+    return pageUrl;
+  }
+}

server/src/adapters/DongraysAdapter.ts

@@ -0,0 +1,80 @@
+import type { SearchAdapter, MIDICandidate } from '../types.js';
+import { ScoreUtils } from '../utils/ScoreUtils.js';
+
+export class DongraysAdapter implements SearchAdapter {
+  name = 'dongrays';
+  private baseUrl = 'https://www.dongrays.net';
+
+  async search(query: string): Promise<MIDICandidate[]> {
+    try {
+      // Dongrays search endpoint (simplified for MVP)
+      const searchUrl = `${this.baseUrl}/search?q=${encodeURIComponent(query)}`;
+      const response = await fetch(searchUrl, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; MotifBot/1.0)'
+        }
+      });
+
+      if (!response.ok) {
+        throw new Error(`Dongrays search failed: ${response.status}`);
+      }
+
+      const html = await response.text();
+      return this.parseSearchResults(html, query);
+    } catch (error) {
+      console.error('Dongrays search error:', error);
+      return [];
+    }
+  }
+
+  private parseSearchResults(html: string, query: string): MIDICandidate[] {
+    const candidates: MIDICandidate[] = [];
+    
+    // Look for .mid file links in the HTML
+    const midiRegex = /<a[^>]*href="([^"]*\.mid)"[^>]*>([^<]*)</gi;
+    let match;
+
+    while ((match = midiRegex.exec(html)) !== null && candidates.length < 10) {
+      const [, href, title] = match;
+      
+      if (href && title.trim()) {
+        const fullUrl = href.startsWith('http') ? href : `${this.baseUrl}${href}`;
+        const confidence = ScoreUtils.calculateConfidence(title, query, this.name);
+        
+        candidates.push({
+          id: `dongrays_${Buffer.from(fullUrl).toString('base64').slice(0, 16)}`,
+          title: title.trim(),
+          source: 'dongrays',
+          pageUrl: fullUrl,
+          midiUrl: fullUrl, // Direct link to MIDI file
+          confidence
+        });
+      }
+    }
+
+    // Also look for download links that might contain MIDI files
+    const downloadRegex = /<a[^>]*href="([^"]*download[^"]*)"[^>]*>([^<]*mid[^<]*)</gi;
+    while ((match = downloadRegex.exec(html)) !== null && candidates.length < 10) {
+      const [, href, title] = match;
+      
+      if (href && title.trim()) {
+        const fullUrl = href.startsWith('http') ? href : `${this.baseUrl}${href}`;
+        const confidence = ScoreUtils.calculateConfidence(title, query, this.name);
+        
+        candidates.push({
+          id: `dongrays_dl_${Buffer.from(fullUrl).toString('base64').slice(0, 16)}`,
+          title: title.trim(),
+          source: 'dongrays',
+          pageUrl: fullUrl,
+          midiUrl: fullUrl,
+          confidence: confidence * 0.8 // Slightly lower confidence for download links
+        });
+      }
+    }
+
+    return candidates
+      .filter(c => c.confidence > 0.3)
+      .sort((a, b) => b.confidence - a.confidence)
+      .slice(0, 5);
+  }
+}

server/src/server.ts

@@ -0,0 +1,63 @@
+import express from 'express';
+import cors from 'cors';
+import { MIDISearchService } from './services/MIDISearchService.js';
+import { MIDIFetchService } from './services/MIDIFetchService.js';
+
+const app = express();
+const port = process.env.PORT || 3001;
+
+app.use(cors());
+app.use(express.json());
+
+const searchService = new MIDISearchService();
+const fetchService = new MIDIFetchService();
+
+// Search for MIDI files
+app.get('/api/midi/search', async (req, res) => {
+  try {
+    const query = req.query.q as string;
+    if (!query) {
+      return res.status(400).json({ error: 'Query parameter "q" is required' });
+    }
+
+    console.log(`Searching for: ${query}`);
+    const results = await searchService.search(query);
+    res.json({ results, count: results.length });
+  } catch (error) {
+    console.error('Search error:', error);
+    res.status(500).json({ error: 'Search failed' });
+  }
+});
+
+// Fetch and proxy MIDI file
+app.get('/api/midi/fetch', async (req, res) => {
+  try {
+    const url = req.query.u as string;
+    if (!url) {
+      return res.status(400).json({ error: 'URL parameter "u" is required' });
+    }
+
+    console.log(`Fetching: ${url}`);
+    const result = await fetchService.fetch(url);
+    
+    if (result.success) {
+      res.setHeader('Content-Type', 'audio/midi');
+      res.setHeader('Content-Length', result.data!.length);
+      res.send(result.data);
+    } else {
+      res.status(404).json({ error: result.error });
+    }
+  } catch (error) {
+    console.error('Fetch error:', error);
+    res.status(500).json({ error: 'Fetch failed' });
+  }
+});
+
+// Health check
+app.get('/health', (req, res) => {
+  res.json({ status: 'ok', timestamp: new Date().toISOString() });
+});
+
+app.listen(port, () => {
+  console.log(`🎵 Motif backend running on port ${port}`);
+});

server/src/services/MIDIFetchService.ts

@@ -0,0 +1,163 @@
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+import { ScoreUtils } from '../utils/ScoreUtils.js';
+import type { CacheEntry } from '../types.js';
+
+export class MIDIFetchService {
+  private cacheDir = path.join(process.cwd(), 'cache');
+  private cacheIndex = new Map<string, CacheEntry>();
+
+  constructor() {
+    this.initializeCache();
+  }
+
+  async fetch(url: string): Promise<{ success: boolean; data?: ArrayBuffer; error?: string }> {
+    try {
+      // Check cache first
+      const hash = this.hashUrl(url);
+      const cached = await this.getCached(hash);
+      if (cached) {
+        console.log(`Cache hit for ${url}`);
+        return { success: true, data: cached };
+      }
+
+      // Fetch from network
+      console.log(`Fetching from network: ${url}`);
+      const response = await fetch(url, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; MotifBot/1.0)',
+        },
+        signal: AbortSignal.timeout(10000) // 10 second timeout
+      });
+
+      if (!response.ok) {
+        return { success: false, error: `HTTP ${response.status}` };
+      }
+
+      const buffer = await response.arrayBuffer();
+      
+      // Validate file
+      const validation = this.validateMIDI(buffer);
+      if (!validation.valid) {
+        return { success: false, error: validation.error };
+      }
+
+      // Quality check
+      const quality = ScoreUtils.assessQuality(buffer);
+      if (quality.score < 0.3) {
+        return { 
+          success: false, 
+          error: `Poor quality: ${quality.issues.join(', ')}` 
+        };
+      }
+
+      // Cache the file
+      await this.cacheFile(hash, buffer, url);
+      
+      return { success: true, data: buffer };
+      
+    } catch (error) {
+      console.error('Fetch error:', error);
+      return { 
+        success: false, 
+        error: error instanceof Error ? error.message : 'Unknown error' 
+      };
+    }
+  }
+
+  private validateMIDI(buffer: ArrayBuffer): { valid: boolean; error?: string } {
+    if (buffer.byteLength < 14) {
+      return { valid: false, error: 'File too small to be valid MIDI' };
+    }
+
+    // Check MIDI header
+    const view = new Uint8Array(buffer);
+    const header = String.fromCharCode(...view.slice(0, 4));
+    
+    if (header !== 'MThd') {
+      return { valid: false, error: 'Invalid MIDI header' };
+    }
+
+    // Size limits
+    if (buffer.byteLength > 10_000_000) { // 10MB max
+      return { valid: false, error: 'File too large' };
+    }
+
+    return { valid: true };
+  }
+
+  private hashUrl(url: string): string {
+    return crypto.createHash('sha256').update(url).digest('hex').slice(0, 16);
+  }
+
+  private async initializeCache(): Promise<void> {
+    try {
+      await fs.mkdir(this.cacheDir, { recursive: true });
+      
+      // Load cache index if it exists
+      const indexPath = path.join(this.cacheDir, 'index.json');
+      try {
+        const indexData = await fs.readFile(indexPath, 'utf-8');
+        const entries = JSON.parse(indexData) as CacheEntry[];
+        for (const entry of entries) {
+          this.cacheIndex.set(entry.hash, entry);
+        }
+        console.log(`Loaded ${entries.length} cache entries`);
+      } catch {
+        // Index doesn't exist yet, that's fine
+      }
+    } catch (error) {
+      console.error('Cache initialization failed:', error);
+    }
+  }
+
+  private async getCached(hash: string): Promise<ArrayBuffer | null> {
+    const entry = this.cacheIndex.get(hash);
+    if (!entry) return null;
+
+    // Check if file still exists
+    const filePath = path.join(this.cacheDir, entry.filename);
+    try {
+      const buffer = await fs.readFile(filePath);
+      return buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength);
+    } catch {
+      // File doesn't exist, remove from index
+      this.cacheIndex.delete(hash);
+      return null;
+    }
+  }
+
+  private async cacheFile(hash: string, buffer: ArrayBuffer, originalUrl: string): Promise<void> {
+    try {
+      const filename = `${hash}.mid`;
+      const filePath = path.join(this.cacheDir, filename);
+      
+      await fs.writeFile(filePath, new Uint8Array(buffer));
+      
+      const entry: CacheEntry = {
+        hash,
+        filename,
+        size: buffer.byteLength,
+        timestamp: Date.now()
+      };
+      
+      this.cacheIndex.set(hash, entry);
+      await this.saveIndex();
+      
+      console.log(`Cached ${originalUrl} as ${filename} (${buffer.byteLength} bytes)`);
+    } catch (error) {
+      console.error('Cache write failed:', error);
+    }
+  }
+
+  private async saveIndex(): Promise<void> {
+    try {
+      const indexPath = path.join(this.cacheDir, 'index.json');
+      const entries = Array.from(this.cacheIndex.values());
+      await fs.writeFile(indexPath, JSON.stringify(entries, null, 2));
+    } catch (error) {
+      console.error('Cache index save failed:', error);
+    }
+  }
+}

server/src/services/MIDISearchService.ts

@@ -0,0 +1,63 @@
+import type { SearchAdapter, MIDICandidate } from '../types.js';
+import { BitMidiAdapter } from '../adapters/BitMidiAdapter.js';
+import { DongraysAdapter } from '../adapters/DongraysAdapter.js';
+
+export class MIDISearchService {
+  private adapters: SearchAdapter[];
+
+  constructor() {
+    this.adapters = [
+      new BitMidiAdapter(),
+      new DongraysAdapter()
+    ];
+  }
+
+  async search(query: string): Promise<MIDICandidate[]> {
+    const allResults: MIDICandidate[] = [];
+    
+    // Search all adapters in parallel
+    const searchPromises = this.adapters.map(async adapter => {
+      try {
+        const results = await adapter.search(query);
+        console.log(`${adapter.name}: Found ${results.length} results`);
+        return results;
+      } catch (error) {
+        console.error(`${adapter.name} search failed:`, error);
+        return [];
+      }
+    });
+
+    const results = await Promise.all(searchPromises);
+    
+    // Combine and deduplicate results
+    for (const adapterResults of results) {
+      allResults.push(...adapterResults);
+    }
+
+    // Remove duplicates (same MIDI URL)
+    const uniqueResults = this.deduplicateResults(allResults);
+    
+    // Sort by confidence score
+    uniqueResults.sort((a, b) => b.confidence - a.confidence);
+    
+    // Return top 10 results
+    return uniqueResults.slice(0, 10);
+  }
+
+  private deduplicateResults(results: MIDICandidate[]): MIDICandidate[] {
+    const seen = new Set<string>();
+    const unique: MIDICandidate[] = [];
+
+    for (const result of results) {
+      // Create dedup key from MIDI URL or title+source
+      const key = result.midiUrl || `${result.title}_${result.source}`;
+      
+      if (!seen.has(key)) {
+        seen.add(key);
+        unique.push(result);
+      }
+    }
+
+    return unique;
+  }
+}

server/src/types.ts

@@ -0,0 +1,22 @@
+export interface MIDICandidate {
+  id: string;
+  title: string;
+  source: 'bitmidi' | 'dongrays';
+  pageUrl: string;
+  midiUrl: string;
+  confidence: number;
+  fileSize?: number;
+  duration?: number;
+}
+
+export interface SearchAdapter {
+  name: string;
+  search(query: string): Promise<MIDICandidate[]>;
+}
+
+export interface CacheEntry {
+  hash: string;
+  filename: string;
+  size: number;
+  timestamp: number;
+}

server/src/utils/ScoreUtils.ts

@@ -0,0 +1,109 @@
+export class ScoreUtils {
+  static calculateConfidence(title: string, query: string, source: string): number {
+    const titleLower = title.toLowerCase();
+    const queryLower = query.toLowerCase();
+    
+    let score = 0;
+    
+    // Token matching - split and check individual words
+    const titleTokens = this.tokenize(titleLower);
+    const queryTokens = this.tokenize(queryLower);
+    
+    // Exact title match gets high score
+    if (titleLower.includes(queryLower)) {
+      score += 0.8;
+    }
+    
+    // Token overlap scoring
+    const matchingTokens = queryTokens.filter(token => 
+      titleTokens.some(titleToken => 
+        titleToken.includes(token) || token.includes(titleToken)
+      )
+    );
+    
+    const tokenMatchRatio = matchingTokens.length / queryTokens.length;
+    score += tokenMatchRatio * 0.6;
+    
+    // Penalty for low-quality indicators
+    const penalties = [
+      { pattern: /karaoke|kar|midkar/, penalty: 0.3 },
+      { pattern: /vocal|lyrics/, penalty: 0.2 },
+      { pattern: /demo|test|sample/, penalty: 0.2 },
+      { pattern: /incomplete|broken/, penalty: 0.5 },
+    ];
+    
+    for (const { pattern, penalty } of penalties) {
+      if (pattern.test(titleLower)) {
+        score -= penalty;
+      }
+    }
+    
+    // Bonus for direct .mid links
+    if (title.includes('.mid')) {
+      score += 0.1;
+    }
+    
+    // Source-specific scoring adjustments
+    if (source === 'bitmidi') {
+      score += 0.1; // Slight preference for BitMidi (more curated)
+    }
+    
+    // Ensure score is between 0 and 1
+    return Math.max(0, Math.min(1, score));
+  }
+  
+  static assessQuality(buffer: ArrayBuffer): { 
+    score: number; 
+    duration?: number; 
+    trackCount?: number; 
+    issues: string[] 
+  } {
+    const issues: string[] = [];
+    let score = 0.5; // Base score
+    
+    // File size checks
+    const size = buffer.byteLength;
+    if (size < 1000) {
+      issues.push('File too small');
+      score -= 0.3;
+    } else if (size > 5_000_000) {
+      issues.push('File very large');
+      score -= 0.1;
+    } else {
+      score += 0.1; // Good size range
+    }
+    
+    // Basic MIDI header validation
+    const view = new Uint8Array(buffer);
+    const header = String.fromCharCode(...view.slice(0, 4));
+    
+    if (header !== 'MThd') {
+      issues.push('Invalid MIDI header');
+      score -= 0.5;
+    } else {
+      score += 0.2;
+    }
+    
+    // TODO: More sophisticated parsing for duration, tempo changes, track count
+    // For MVP, we'll do basic heuristics
+    
+    return {
+      score: Math.max(0, Math.min(1, score)),
+      issues
+    };
+  }
+  
+  private static tokenize(text: string): string[] {
+    return text
+      .toLowerCase()
+      .replace(/[^\w\s]/g, ' ') // Remove punctuation
+      .split(/\s+/)
+      .filter(token => token.length > 1) // Remove single characters
+      .filter(token => !this.isStopWord(token));
+  }
+  
+  private static isStopWord(word: string): boolean {
+    const stopWords = ['the', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'];
+    return stopWords.includes(word);
+  }
+}

server/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "strict": true,
+    "skipLibCheck": true,
+    "outDir": "dist",
+    "rootDir": "src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}