256bit.org Blog

mirror(){
 # Mirror Webpages using wget
 # By default they will be stored under ~/mirror,
 # unless $MIRROR_PATH is set

 case $1 in
     --help|-h|-?)
         cat <<-EOF
         A Shell function that will mirror the given URLs in a local
         directory. By default this is ~/mirror, but this can be changed
         by setting MIRROR_PATH to the desired directory.
         It will use wget to mirror the URLs so wget is expected to work.
         Usage: mirror <URL>
         EOF
         ;;
     *)
      if [ -z "$@" ]; then
         echo 'Please enter URL or see help (-h|--help|-?)'
     else
         MIRROR_PATH=${MIRROR_PATH:-~/mirror}
         [ -d $MIRROR_PATH ] || mkdir $MIRROR_PATH
             for url in "$@"
             do
                 dir="${MIRROR_PATH}/${url#http://}"
                 if [ -d "$dir" ]; then
                     echo "Updating $dir"
                 else
                     echo "Retrieving $url under $dir"
                 fi
                 wget -q -N -nd -k -p -H -P "$dir" "$url"
                 echo -e "Date: $(date -R)\tURL: $url"> "$dir/doc-get-info.txt"
             done
     fi
     ;;
 esac
 }