SELECT *
FROM (
SELECT DISTINCT
    magicDSRun.magic_ds_id,
    magicRun.magic_id,
    magicRun.exp_id,
    camera,
    magicMask.uri as streaks_uri,
    stage,
    stage_id,
    class_id as component,
    rawImfile.uri as uri,
    NULL as path_base,
    camProcessedExp.path_base as cam_path_base,
    outroot,
    recoveryroot,
    re_place,
    remove
FROM magicDSRun
JOIN magicMask USING (magic_id)
JOIN magicRun USING (magic_id)
JOIN camProcessedExp USING(cam_id)
JOIN rawImfile ON magicRun.exp_id = rawImfile.exp_id
LEFT JOIN magicDSFile
    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    AND magicDSFile.component = rawImfile.class_id
WHERE
    magicDSRun.state = 'run'
    AND magicDSRun.stage = 'raw'
    AND magicDSFile.component IS NULL
UNION
  -- chipProcesedImfiles
SELECT DISTINCT
    magicDSRun.magic_ds_id,
    magicDSRun.magic_id,
    chipRun.exp_id,
    camera,
    magicMask.uri as streaks_uri,
    stage,
    stage_id,
    class_id as component,
    chipProcessedImfile.uri,
    chipProcessedImfile.path_base,
    camProcessedExp.path_base as cam_path_base,
    outroot,
    recoveryroot,
    re_place,
    remove
FROM magicDSRun
JOIN magicMask USING (magic_id)
JOIN camProcessedExp USING(cam_id)
JOIN chipRun ON chip_id = stage_id
JOIN chipProcessedImfile USING(chip_id)
JOIN rawExp ON chipRun.exp_id = rawExp.exp_id
LEFT JOIN magicDSFile
    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    AND magicDSFile.component = chipProcessedImfile.class_id
WHERE
    magicDSRun.state = 'run'
    AND magicDSRun.stage = 'chip'
    AND chipRun.state = 'full'
    AND chipProcessedImfile.fault = 0
    AND magicDSFile.component IS NULL
UNION
-- warpSkyfiles
SELECT DISTINCT
    magicDSRun.magic_ds_id,
    magicRun.magic_id,
    magicRun.exp_id,
    camera,
    magicMask.uri as streaks_uri,
    stage,
    stage_id,
    warpSkyfile.skycell_id as component,
    warpSkyfile.uri,
    warpSkyfile.path_base,
    NULL as cam_path_base,
    outroot,
    recoveryroot,
    re_place,
    remove
FROM magicDSRun
JOIN magicMask USING (magic_id)
JOIN magicRun USING (magic_id)
JOIN magicInputSkyfile USING(magic_id)
JOIN warpSkyfile ON warp_id = stage_id
JOIN warpRun USING(warp_id)
JOIN rawExp ON magicRun.exp_id = rawExp.exp_id
LEFT JOIN magicDSFile
    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    AND magicDSFile.component = warpSkyfile.skycell_id
WHERE
    magicDSRun.state = 'run'
    AND magicDSRun.stage = 'warp'
    AND warpRun.state = 'full'
    AND warpSkyfile.fault = 0
    AND warpSkyfile.ignored = 0
    AND magicDSFile.component IS NULL
UNION
-- diffSkyfiles
SELECT DISTINCT
    magicDSRun.magic_ds_id,
    magicRun.magic_id,
    magicRun.exp_id,
    rawExp.camera,
    magicMask.uri as streaks_uri,
    stage,
    magicRun.diff_id as stage_id,
    diffSkyfile.skycell_id as component,
    diffSkyfile.uri,
    diffSkyfile.path_base,
    NULL as cam_path_base,
    outroot,
    recoveryroot,
    re_place,
    remove
FROM rawExp
JOIN magicRun USING (exp_id)
JOIN magicMask USING (magic_id)
JOIN magicDSRun USING(magic_id)
JOIN magicInputSkyfile USING(magic_id)
JOIN diffSkyfile
    ON  magicInputSkyfile.diff_id = diffSkyfile.diff_id
    AND magicInputSkyfile.node = diffSkyfile.skycell_id
LEFT JOIN magicDSFile
    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
    AND magicDSFile.component = diffSkyfile.skycell_id
WHERE
    magicDSRun.state = 'run'
    AND magicDSRun.stage = 'diff'
    AND diffSkyfile.fault = 0
    AND magicDSFile.component IS NULL
) as Foo
