param([Parameter(Mandatory=$true)][string] $fname)

function make-ouf($ofn, $ofe)
{
   new-object IO.FileInfo("output/$ofn.$ofe")
}

function log($text)
{
   out-file -append log.txt -inputobject "$text`n" -encoding ascii -nonewline
   echo "*** $text"
}

if(!(test-path output)) {
   mkdir output
}

$uagent = "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0"
$progressPreference = "silentlyContinue"
$sums = @()
$numerrors = 0
$duplicates = 0
$newfiles = 0

if(test-path "hashes.txt") {
   $sums = (get-content "hashes.txt") -split '\n'
} else {
   log "WARNING: No hashes file. Duplicates will not be removed."
}

log "Started download at $(date)"

foreach($line in [IO.File]::ReadLines((resolve-path $fname))) {
   $inf = $line.split('/')[-1].split(':')[0].split('.')
   $ofn = $inf[0 .. ($inf.length - 2)] -join '.'
   $ofe = $inf[-1]
   $ouf = (make-ouf $ofn $ofe)

   while($ouf.exists) {
      $ofn += '_'
      $ouf = (make-ouf $ofn $ofe)
   }

   echo ">>\ $line"
   echo "  | $ouf"

   try {
      $uri = [Uri]$line
      $r = invoke-webrequest -useragent $uagent -header @{"Referer" = ('http://' + $uri.host) } -uri $line -sessionvariable ofp -erroraction stop -timeout 60 -outfile $ouf
      $sum = ((xxhsum $ouf) -split " ")[0]
      if($sums.contains($sum)) {
         rm $ouf
         $duplicates += 1
         echo "  / duplicate $sum"
      } else {
         $sums += $sum
         $newfiles += 1
         echo "<</ success $sum"
      }
   } catch {
      echo "!!! error: " $error[0]
      log "Failed: $line"
      $numerrors += 1
   }
}

if($numerrors)  {log "$numerrors errors"}              else {log "No errors"}
if($duplicates) {log "$duplicates duplicates removed"} else {log "No duplicates"}
if($newfiles)   {log "$newfiles new files"}            else {log "No new files"}

copy-item hashes.txt hashes.old.txt
set-content -path hashes.txt -value $sums

log "Done."