Auto-Labeling & Annotations

{
  "type": "image",
  "objects": [
    { "label": "mailbox", "confidence": 0.95, "position": "center", "approximate_coverage": "25% of frame" },
    { "label": "house", "confidence": 0.88, "position": "upper-right", "approximate_coverage": "35% of frame" }
  ],
  "scene": { "setting": "outdoor residential neighborhood", "indoor": false, "confidence": 0.92 },
  "text_extraction": {
    "full_text": "1234 Oak Street",
    "words": [
      { "text": "1234", "confidence": 0.97 },
      { "text": "Oak", "confidence": 0.95 },
      { "text": "Street", "confidence": 0.93 }
    ]
  },
  "color_palette": ["#8B4513", "#228B22", "#87CEEB"],
  "composition": "centered mailbox with residential background, natural daylight",
  "face_count": 0,
  "orientation": "landscape",
  "quality_metrics": { "blur_score": 0.08, "exposure": "normal", "noise_level": "low" },
  "safety": { "nsfw_score": 0.0, "violence_score": 0.0, "pii_detected": false },
  "annotation_model": "claude-sonnet-4-20250514"
}

{
  "text": "The battery lasts about 8 hours.",
  "start_seconds": 0.0,
  "end_seconds": 2.34,
  "confidence": 0.82
}

"word_timestamps": [
  { "word": "The", "start": 0.0, "end": 0.12 },
  { "word": "battery", "start": 0.12, "end": 0.56 },
  { "word": "lasts", "start": 0.56, "end": 0.89 }
]

{
  "type": "audio",
  "duration_seconds": 21.45,
  "sample_rate": 44100,
  "bitrate_kbps": 128,
  "codec": "aac",
  "channels": 1,
  "speaker_count": 1,
  "language": "en",
  "topics": ["product review", "technology"],
  "keywords": ["battery life", "screen quality"],
  "noise_level": "low",
  "snr_db": 32,
  "emotion_tone": {
    "primary": "neutral",
    "confidence": 0.85,
    "secondary": "calm"
  },
  "room_acoustics": {
    "estimated_room_size": "medium",
    "reverb_level": "slight",
    "estimated_rt60_seconds": 0.4
  },
  "background_sounds": [
    { "label": "HVAC", "confidence": 0.75, "prominence": "faint" }
  ],
  "speaker_demographics": {
    "estimated_gender": "male",
    "estimated_age_range": "adult",
    "accent_region": "us_general"
  },
  "transcript_segments": [
    { "text": "The battery lasts about 8 hours.", "start_seconds": 0.0, "end_seconds": 2.34, "confidence": 0.82 },
    { "text": "Screen quality is excellent.", "start_seconds": 2.34, "end_seconds": 4.87, "confidence": 0.79 }
  ],
  "word_timestamps": [
    { "word": "The", "start": 0.0, "end": 0.12 },
    { "word": "battery", "start": 0.12, "end": 0.56 },
    { "word": "lasts", "start": 0.56, "end": 0.89 },
    { "word": "about", "start": 0.89, "end": 1.15 },
    { "word": "8", "start": 1.15, "end": 1.32 },
    { "word": "hours", "start": 1.32, "end": 1.78 }
  ],
  "annotation_model": "claude-sonnet-4-20250514"
}

{
  "type": "video",
  "duration_seconds": 15.2,
  "width": 1920,
  "height": 1080,
  "fps": 30,
  "video_codec": "h264",
  "audio_codec": "aac",
  "scenes": [
    { "description": "Entrance area with door", "start_seconds": 0.0, "end_seconds": 5.2, "confidence": 0.88 },
    { "description": "Living room with furniture", "start_seconds": 5.2, "end_seconds": 10.5, "confidence": 0.92 },
    { "description": "Kitchen area", "start_seconds": 10.5, "end_seconds": 15.2, "confidence": 0.85 }
  ],
  "actions": ["walking", "panning camera", "moving through rooms"],
  "object_tracking": [
    { "label": "furniture", "confidence": 0.90, "appears_in_scenes": [0, 1, 2] },
    { "label": "television", "confidence": 0.88, "appears_in_scenes": [1] }
  ],
  "keyframe_descriptions": ["Doorway with wooden frame", "Bright living room with windows", "Kitchen counter with appliances"],
  "face_count": 0,
  "speaker_count": 1,
  "noise_level": "low",
  "quality_metrics": { "blur_score": 0.1, "exposure": "normal", "stability": "stable" },
  "safety": { "nsfw_score": 0.0, "violence_score": 0.0 },
  "transcript_segments": [
    { "text": "Here's a walkthrough of the apartment.", "start_seconds": 1.0, "end_seconds": 3.2 }
  ],
  "word_timestamps": [
    { "word": "Here's", "start": 1.0, "end": 1.25 },
    { "word": "a", "start": 1.25, "end": 1.35 },
    { "word": "walkthrough", "start": 1.35, "end": 1.98 }
  ],
  "annotation_model": "claude-sonnet-4-20250514"
}

curl -H "Authorization: Bearer fh_live_..." \
  https://api.firsthandapi.com/v1/jobs/job_01JQ.../files

const files = await client.getJobFiles('job_01JQ...');
 
for (const file of files.data) {
  if (file.annotations?.type === 'image') {
    console.log('Objects:', file.annotations.objects.map(o => `${o.label} (${o.confidence})`));
    console.log('OCR:', file.annotations.text_extraction?.full_text);
    console.log('Faces:', file.annotations.face_count);
    console.log('NSFW score:', file.annotations.safety.nsfw_score);
  }
  if (file.annotations?.type === 'audio') {
    console.log('Duration:', file.annotations.duration_seconds, 's');
    console.log('Sample rate:', file.annotations.sample_rate, 'Hz');
    console.log('Speaker:', file.annotations.speaker_demographics);
    console.log('Words:', file.annotations.word_timestamps?.length);
  }
}

"keyframes": [
  { "index": 0, "download_url": "https://...", "download_url_expires_at": "2026-04-10T..." },
  { "index": 1, "download_url": "https://...", "download_url_expires_at": "2026-04-10T..." },
  { "index": 2, "download_url": "https://...", "download_url_expires_at": "2026-04-10T..." }
]