75 lines
1.9 KiB
PowerShell
Executable File
75 lines
1.9 KiB
PowerShell
Executable File
param([Parameter(Mandatory=$true)][string] $fname)
|
|
|
|
function make-ouf($ofn, $ofe)
|
|
{
|
|
new-object IO.FileInfo("output/$ofn.$ofe")
|
|
}
|
|
|
|
function log($text)
|
|
{
|
|
out-file -append log.txt -inputobject "$text`n" -encoding ascii -nonewline
|
|
echo "*** $text"
|
|
}
|
|
|
|
if(!(test-path output)) {
|
|
mkdir output
|
|
}
|
|
|
|
$uagent = "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0"
|
|
$progressPreference = "silentlyContinue"
|
|
$sums = @()
|
|
$numerrors = 0
|
|
$duplicates = 0
|
|
$newfiles = 0
|
|
|
|
if(test-path "hashes.txt") {
|
|
$sums = (get-content "hashes.txt") -split '\n'
|
|
} else {
|
|
log "WARNING: No hashes file. Duplicates will not be removed."
|
|
}
|
|
|
|
log "Started download at $(date)"
|
|
|
|
foreach($line in [IO.File]::ReadLines((resolve-path $fname))) {
|
|
$inf = $line.split('/')[-1].split(':')[0].split('.')
|
|
$ofn = $inf[0 .. ($inf.length - 2)] -join '.'
|
|
$ofe = $inf[-1]
|
|
$ouf = (make-ouf $ofn $ofe)
|
|
|
|
while($ouf.exists) {
|
|
$ofn += '_'
|
|
$ouf = (make-ouf $ofn $ofe)
|
|
}
|
|
|
|
echo ">>\ $line"
|
|
echo " | $ouf"
|
|
|
|
try {
|
|
$uri = [Uri]$line
|
|
$r = invoke-webrequest -useragent $uagent -header @{"Referer" = ('http://' + $uri.host) } -uri $line -sessionvariable ofp -erroraction stop -timeout 60 -outfile $ouf
|
|
$sum = ((xxhsum $ouf) -split " ")[0]
|
|
if($sums.contains($sum)) {
|
|
rm $ouf
|
|
$duplicates += 1
|
|
echo " / duplicate $sum"
|
|
} else {
|
|
$sums += $sum
|
|
$newfiles += 1
|
|
echo "<</ success $sum"
|
|
}
|
|
} catch {
|
|
echo "!!! error: " $error[0]
|
|
log "Failed: $line"
|
|
$numerrors += 1
|
|
}
|
|
}
|
|
|
|
if($numerrors) {log "$numerrors errors"} else {log "No errors"}
|
|
if($duplicates) {log "$duplicates duplicates removed"} else {log "No duplicates"}
|
|
if($newfiles) {log "$newfiles new files"} else {log "No new files"}
|
|
|
|
copy-item hashes.txt hashes.old.txt
|
|
set-content -path hashes.txt -value $sums
|
|
|
|
log "Done."
|