# # (gH)   -_-  getSra.pl  ;  TimeStamp (unix) : 13 Juin 2016 vers 18:44

#################################################################################
#
# this script download all the sra files found in the parameter
#
# for instance, if the files mySrafiles.txt contains
#
#   ftp://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByExp/sra/SRX/SRX101/SRX1011066/SRR1999023/SRR1999023.sra
#   ftp://ftp-trace.ncbi.nlm.nih.gov/sra/sra-instant/reads/ByExp/sra/SRX/SRX101/SRX1011065/SRR1999022/SRR1999022.sra
#
# the script will download these two files
#
#
#################################################################################

print "\n getSra.pl (gH) version 1.03\n" ;

use strict ;
use warnings ;
use POSIX qw/ceil/;

if ($#ARGV==-1) {
  print "\n" ;
  print " the script getSra.pl download all the files found in the parameter file \n" ;
  print " syntax   : perl getSra.pl FILE      \n" ;
  print " example  : perl getSra.pl mySraFiles.txt \n" ;
  print "           where mySraFiles.txt contains lines like \n" ;
  print "               ftp://ftp-trace.ncbi.nlm.nih.gov/sra/.../SRR1999023.sra" ;
  print "\n\n" ;
  exit(-1) ;
} ; # end if

# store date and time to be able to compute duration

my $date1 = &dateEtHeure() ;
my $time1 = time() ;
system("echo $date1 > time.start") ;

####################################################################
####################################################################

# let's check the parameter file is present

my $inFile = $ARGV[0] ;

if (!-e $inFile) {
  print " file $inFile not found.STOP.\n\n" ;
  exit(-2) ;
} ; # end if

####################################################################
####################################################################

# 1. get the expected size of the files

####################################################################
####################################################################

print "\n" ;
print " 1. retrieving the file sizes for $inFile.\n\n" ;
print "\n" ;

my $expsFile = "expected_sizes.txt" ;

open(ESFILE,">$expsFile") or die ("\n\n! Unable to write to $expsFile\n\n") ;
print ESFILE "Start: ".&dateEtHeure()."\n" ;

# loop over all the URLs and get the sizes using wget --spider

open(INFILE,"<$inFile")   or die ("\n\n! Unable to read from $inFile\n\n") ;

my $nbf = 0 ; # nombre de fichiers

while (my $lig=<INFILE>) {
  $nbf++ ;
} ; # fin de tant que
close(INFILE) ;

my @tdm     ; # tableau des mots
my @wgTdm   ; # tableau des mots
my $wgetLst = "wget.lst" ;
my $wgetErr = "wget_err.lst" ;
my $resCmd  = "" ;
my $fSize   = 0 ;
my $tailleh = 0 ;
my $total   = 0 ;
my $totalh  = 0 ;
my $ligRes   = "" ;
my $ligRes1  = "" ;
my $ligRes2  = "" ;
my $ligRes3  = "" ;
my $ligRes4  = "" ;
my $ligRes5  = "" ;

open(INFILE,"<$inFile")   or die ("\n\n! Unable to read from $inFile\n\n") ;
my $idf = 0 ; # nombre de fichiers
while (my $lig=<INFILE>) {
  chomp($lig) ;
  $idf++ ;
  $ligRes1 = "file ".sprintf("%4d",$idf)." / $nbf : " ;
  print $ligRes1 ;
  @tdm =  split(/\//,$lig) ;
  $ligRes2 = $tdm[$#tdm] ;
  print $ligRes2 ;
  $resCmd = `wget --spider $lig > $wgetLst 2> $wgetLst ` ;
  open(WGFILE,"<$wgetLst")   or die ("\n\n! Unable to read from $wgetLst\n\n") ;
  while (my $wgLine = <WGFILE>) {
    @wgTdm = split(/\s+/,$wgLine) ;
    if ($wgTdm[1]) {
    if ($wgTdm[1] eq 'SIZE') {
       print "  ".$wgTdm[1]." " ;
       $fSize   = 0 ;
       $tailleh = 0 ;
       if ($wgTdm[4] =~ /[0-9]+/) {
          $fSize = $wgTdm[4] ;
          $tailleh =  &tailleH($fSize) ;
       } ; # fin si sur numérique
       $ligRes3  = sprintf("%12s",$fSize) ;
       $ligRes3 .= sprintf("%10s",$tailleh) ;
       print $ligRes3 ;
       $total += $fSize ;
    } ; # fin de si sur size
    } ; # fin de si sur non vide
  } ; # fin de tant que
  close(WGFILE) ;
  $ligRes4 = "  -- total " ;
  print $ligRes4 ;
  $totalh =  &tailleH($total) ;
  $ligRes5 = sprintf("%12s",$total).sprintf("%10s",$totalh)."\n" ;
  print $ligRes5 ;
  print ESFILE $ligRes1.$ligRes2.$ligRes3.$ligRes4.$ligRes5 ;
} ; # fin de tant que

close(INFILE) ;

print ESFILE "-- all $nbf file sizes retrieved.\n" ;
print ESFILE "End: ".&dateEtHeure()."\n" ;
close(ESFILE) ;

print "\n you can consult $expsFile to check the size of the files.\n\n" ;

####################################################################
####################################################################

# 2. wget all the files in a loop + progressive display

####################################################################
####################################################################

print "\n" ;
print " 2. retrieving the $nbf files defined by their URL in $inFile.\n\n" ;
print "\n" ;

my $logFile = "getSra_log.txt" ;
system("rm -rf $logFile");

my $wgTmp = "wget_tmp.txt" ;

# check if K to write to log

open(LOGFILE,">$logFile")    or die ("\n\n! Unable to write in $logFile\n\n") ;
print LOGFILE "getSra.pl : retrieving $nbf files defined in $inFile.\n\n" ;
print LOGFILE "Start: ".&dateEtHeure()."\n" ;
close(LOGFILE) ;

# let's write in the log file for the first time

open(INFILE,"<$inFile")   or die ("\n\n! Unable to read from $inFile\n\n") ;
$idf = 0 ; # nombre de fichiers
while (my $lig=<INFILE>) {
  chomp($lig) ;
  $idf++ ;
  $ligRes1 = &dateEtHeure()."  file ".sprintf("%4d",$idf)." / $nbf : " ;
  print $ligRes1 ;
  @tdm =  split(/\//,$lig) ;
  $ligRes2 = $tdm[$#tdm] ;
  print $ligRes2 ;
  $ligRes3 = " processing...   " ;
  print $ligRes3 ;
  system("wget $lig 2> $wgTmp") ;
  $ligRes4 = "done.\n";
  print $ligRes4 ;
  open(LOGFILE,">>$logFile") or die ("\n\n! Unable to write in $logFile\n\n") ;
  print LOGFILE $ligRes1.$ligRes2.$ligRes3.$ligRes4 ;
  close(LOGFILE) ;
} ; # fin de tant que sur infile

# write in the log file to say we're done

open(LOGFILE,">>$logFile")    or die ("\n\n! Unable to write in $logFile\n\n") ;
print LOGFILE "-- all $nbf files retrieved.\n" ;
print LOGFILE "End: ".&dateEtHeure()."\n\n" ;
close(LOGFILE) ;

####################################################################
####################################################################

# at last, compute global duration

print "\n" ;
my $date2 = &dateEtHeure() ;
system("echo $date2 >  time.stop") ;
my $time2 = time() ;
my $duree = $time2 - $time1 ;
print " Start: $date1 ; end: $date2 ; elapsed time: $duree sec" ;
if ($duree>60) {
  print " (" ;
  my $hrs = int($duree/(24*60.0)) ;
  if ($hrs>0) {
    print "$hrs h " ;
  } ; # fin si
  my $rst = $duree - (24*60.0)*$hrs ;
  my $min = int($rst / 60.0) ;
  my $sec = $rst - 60* $min ;
  print "$min min $sec sec)" ;
} ; # fin si
print ".\n" ;
exit(-1) ;

####################################################################
####################################################################

sub dateEtHeure {

####################################################################

    my ($sec,$min,$hour,$mday,$mmon,$year)=localtime();
    $mmon = $mmon + 1 ;
    $year = $year + 1900 ;
    if (length($sec)<2)  { $sec = "0$sec"   } ;
    if (length($min)<2)  { $min = "0$min"   } ;
    if (length($mday)<2) { $mday = "0$mday" } ;
    if (length($mmon)<2) { $mmon = "0$mmon" } ;
    my $now  = $mday."/".$mmon."/".$year;
    $now .= " ".$hour.":".$min.".".$sec ;

    return $now ;

} ; # fin sub dateEtHeure

####################################################################

sub tailleH {

# affiche lisiblement une taille de fichier

my $nbOctets = $_[0] ;

my @lstUnit = ("","kO","Mo","Go","To","Po") ;

my $p       = 1 ;
my $basep   = 1024 ;
my $limInf  = 1 ;
my $limSup  = $basep**$p ;
my $idu     =  0 ;

while ($nbOctets>$limSup) {
#cat("p=",p," : ",nbOctets,">",limSup,"\n")
#cat("limInf",limInf,",","limSup",limSup,"\n")
#cat("n/limInf",nbOctets/limInf,"\n")
  $idu     = $idu + 1 ;
  $p       = $p+1     ;
  $limInf  = $limSup   ;
  $limSup  = $basep**$p ;
} ; # fin tant que

#cat("donc p=",p," : ",nbOctets,"<",limSup," et idu vaut ",idu,"\n")
#cat("n/limInf",nbOctets/limInf,"\n")
my $th = ($nbOctets*1.0)/$limInf ;
#cat("th=",th,"\n")
$th  = ceil($th) ;
$th .= " $lstUnit[$idu]" ;

return($th)

} ; # fin sub dateEtHeure

####################################################################
#
# normal end of script
#
####################################################################
