#!/bin/ash
#(c) Copyright 2018 Barry Kauler. License: GPLv3 (ref: /usr/share/doc/legal)
#180617 based on 0setup, just for fetching the devuan db and convert to puppy-format.

LANGORG="$(locale | grep '^LANG=' | cut -d '=' -f 2)"
export LANG=C #faster.

BUILD_CHOICE_BOARD='pc' #overridden in build-choices. values: raspi, odroidx, anything else for pc.
if [ -f ./DISTRO_SPECS ];then #running in woofQ
 if [ ! -d sandbox3/rootfs-complete/usr/local/EasyPak ];then
  echo
  echo "Path sandbox3/rootfs-complete/usr/local/EasyPak does not exist, aborting"
  exit
 fi
 SPEC_DS="./DISTRO_SPECS"
. ./DISTRO_SPECS
 [ ! "$DISTRO_DB_SUBNAME" ] && DISTRO_DB_SUBNAME="$DISTRO_COMPAT_VERSION" #fallback if DISTRO_DB_SUBNAME not defined in file DISTRO_SPECS.
. ./rootfs-skeleton/usr/local/EasyPak/DISTRO_SPECS-cut #overrides.
. ./rootfs-skeleton/usr/local/EasyPak/DISTRO_COMPAT_REPOS-devuan
. ./PKGS_MANAGEMENT
. ./build-choices
 RUNNINGPUP='no'
 touch ./PKGS_HOMEPAGES
 cp -a -f ./PKGS_HOMEPAGES sandbox3/rootfs-complete/usr/local/EasyPak/
 cd sandbox3/rootfs-complete/usr/local/EasyPak
else
 SPEC_DS="/etc/DISTRO_SPECS"
 #want to run this script in running puppy, to update db's...
. /etc/DISTRO_SPECS
. /usr/local/EasyPak/DISTRO_SPECS-cut #overrides.
. /usr/local/EasyPak/DISTRO_COMPAT_REPOS-devuan
. /root/.packages/PKGS_MANAGEMENT
. /root/.packages/build-choices
 cd /usr/local/EasyPak
 RUNNINGPUP='yes'
fi

[ -f ./support/debdb2pupdb ] && cp -f ./support/debdb2pupdb /usr/local/petget/ #0setup needs it here, when run from PPM.
#note, 3builddistro copies it into rootfs-complete/usr/local/petget when building a pup.
#note, debdb2pupdb no longer calls find_cat.

#to speed things up, this func replaced by compiled app support/find_cat ...
[ -f ./support/find_cat ] && cp -f ./support/find_cat /usr/local/petget/
FIND_CAT="/usr/local/petget/find_cat"

#make sure have latest in host system...
[ -f rootfs-skeleton/usr/local/petget/categories.dat ] && cp -f rootfs-skeleton/usr/local/petget/categories.dat /usr/local/petget/

mkdir -p /var/woof
RUNDATE="`date`"

#download docs on compatible-distro pkgs...
for PKGLISTSPEC in $PKG_DOCS_DISTRO_COMPAT  #see file DISTRO_PKGS_SPECS-devuan
do
 PKGLISTFILE="`echo -n "$PKGLISTSPEC" | cut -f 3 -d '|'`"
 PKGLISTURI="`echo -n "$PKGLISTSPEC" | cut -f 2 -d '|'`"
 [ "$PKGLISTURI" = "" ] && continue #no url, pkg db file only kept in woof.
 skipdl=""
 if [ "$RUNNINGPUP" = "no" -a -f $PKGLISTFILE ];then
  echo
  echo "Local db file '${PKGLISTFILE}' already exists."
  echo "Press ENTER key only to upgrade it,"
  echo -n "or any other printable char to skip: "
  read skipdl
 fi
 if [ "$skipdl" = "" ];then
  echo
  LANG=${LANGORG} echo "A package information database file needs to be downloaded."
  LANG=${LANGORG} echo "This will be downloaded from:"
  echo "$PKGLISTURI"
  LANG=${LANGORG} echo "and will be processed and named: ${PKGLISTFILE}"
  LANG=${LANGORG} echo -n "Press ENTER key to download, any other to skip it: "
  read downloadit
  if [ "$downloadit" = "" ] ; then
   DLFILE="`basename $PKGLISTURI`"
   [ -f $DLFILE ] && mv -f $DLFILE /tmp/${DLFILE}-backup1 #otherwise wget creates a new file ${DLFILE}.1
   rxvt -name pet -bg orange -geometry 80x10 -e wget $PKGLISTURI
   sync
   xDLFILE="$DLFILE"
   if [ -f $DLFILE ];then
    case "$DLFILE" in
     *.bz2)
      xDLFILE="`basename $DLFILE .bz2`"
      bunzip2 $DLFILE
      RETSTAT=$?
     ;;
     *.gz)
      xDLFILE="`basename $DLFILE .gz`"
      gunzip $DLFILE
      RETSTAT=$?
     ;;
     *.xz)
      xDLFILE="`basename $DLFILE .xz`"
      unxz $DLFILE
      RETSTAT=$?
     ;;
    esac
    if [ $RETSTAT -eq 0 ];then
     LANG=${LANGORG} echo "...success."
     #filter out utf-8 chars...
     iconv -c -f utf-8 -t ascii $xDLFILE > ${PKGLISTFILE}pre
     rm -f $xDLFILE
    else
     LANG=${LANGORG} echo "Downloaded file '${DLFILE}' is corrupted. Deleting file and aborting."
     rm -f $xDLFILE 2>/dev/null
     rm -f $DLFILE 2>/dev/null
     exit
    fi
   else
    LANG=${LANGORG} echo "Failed to download, aborting."
    LANG=${LANGORG} echo "
Failed to download db file: 
 ${PKGLISTURI}
...exited from 0setup script."
    exit
   fi
  else
   LANG=${LANGORG} echo "...skipped"
  fi
 fi
done

# *-update dbs...
DBUPDATEFLAG='invalid'
PKGLISTS_COMPAT_UPDATES=""
if [ "$RUNNINGPUP" == "yes" ];then
 DBUPDATEFLAG='' #yes, update.
else
 echo
 LANG=${LANGORG} echo "Do you want to download the ${DISTRO_BINARY_COMPAT}-updates package databases?"
 LANG=${LANGORG} echo "Accept, they will be downloaded and merged,"
 LANG=${LANGORG} echo "such that only the updated pkg db entries are kept."
 LANG=${LANGORG} echo "Example: as at August 31, 2012, Ubuntu Precise Pangolin 12.04 has the first
'Service Pack' which is version 12.04.1 -- accepting here will obtain the
package lists for 12.04.1 -- it is generally recommended to accept."
 LANG=${LANGORG} echo -n "ENTER key only to accept, any other printable char then ENTER to decline: "
 read DBUPDATEFLAG
fi
if [ "$DBUPDATEFLAG" = "" ];then
 #basically same code as above, slight mods...
 for PKGLISTSPEC in $PKG_DOCS_DISTRO_COMPAT
 do
  [ "`echo -n "$PKGLISTSPEC" | grep 'ppa'`" ] && continue #exclude ppa repo.
  updPTN1="s%${DISTRO_COMPAT_VERSION}%${DISTRO_COMPAT_VERSION}_updates%" #ex: 'precise' becomes 'precise_updates' (underscore because - is delimiter)
  updPTN2="s%${DISTRO_COMPAT_VERSION}%${DISTRO_COMPAT_VERSION}-updates%" #ex: 'precise' becomes 'precise-updates'
  PKGLISTFILE="`echo -n "$PKGLISTSPEC" | cut -f 3 -d '|' | sed "$updPTN1"`"
  PKGLISTURI="`echo -n "$PKGLISTSPEC" | cut -f 2 -d '|' | sed "$updPTN2"`"
  [ "$PKGLISTURI" = "" ] && continue #no url, pkg db file only kept in woof.
  PKGLISTS_COMPAT_UPDATES="${PKGLISTS_COMPAT_UPDATES} ${PKGLISTFILE}" #see further down.
  skipdl=""
  if [ "$RUNNINGPUP" = "no" -a -f $PKGLISTFILE ];then
   echo
   echo "Local db file '${PKGLISTFILE}' already exists."
   echo "Press ENTER key only to upgrade it,"
   echo -n "or any other printable char to skip: "
   read skipdl
  fi
  if [ "$skipdl" = "" ];then
   echo
   LANG=${LANGORG} echo "A package information database file needs to be downloaded."
   LANG=${LANGORG} echo "This will be downloaded from:"
   echo "$PKGLISTURI"
   LANG=${LANGORG} echo "and will be processed and named: ${PKGLISTFILE}"
   LANG=${LANGORG} echo -n "Press ENTER key to download, any other to skip it: "
   read downloadit
   if [ "$downloadit" = "" ] ; then
    DLFILE="`basename $PKGLISTURI`"
    [ -f $DLFILE ] && mv -f $DLFILE /tmp/${DLFILE}-backup1 #otherwise wget creates a new file ${DLFILE}.1
    rxvt -name pet -bg orange -geometry 80x10 -e wget $PKGLISTURI
    sync
    xDLFILE="$DLFILE"
    if [ -f $DLFILE ];then
     case "$DLFILE" in
      *.bz2)
       xDLFILE="`basename $DLFILE .bz2`"
       bunzip2 $DLFILE
       RETSTAT=$?
      ;;
      *.gz)
       xDLFILE="`basename $DLFILE .gz`"
       gunzip $DLFILE
       RETSTAT=$?
      ;;
      *.xz)
       xDLFILE="`basename $DLFILE .xz`"
       unxz $DLFILE
       RETSTAT=$?
      ;;
     esac
     if [ $RETSTAT -eq 0 ];then
      LANG=${LANGORG} echo "...success."
      #remove utf8 chars, cause failure further down.
      iconv -c -f utf-8 -t ascii $xDLFILE > ${PKGLISTFILE}pre #ex: Packages-ubuntu-xenial_updates-mainpre
      rm -f $xDLFILE
      #note, this also works:
      #perl -i.bk -pe 's/[^[:ascii:]]//g;' ${xDLFILE}
      #...orig backed up as ${xDLFILE}.bk
     else
      LANG=${LANGORG} echo "Downloaded file '${DLFILE}' is corrupted. Deleting file and aborting."
      rm -f $xDLFILE 2>/dev/null
      rm -f $DLFILE 2>/dev/null
      exit
     fi
    else
     LANG=${LANGORG} echo "Failed to download, aborting."
     LANG=${LANGORG} echo "
Failed to download db file: 
 $PKGLISTURI
...exited from 0setup script."
     exit
    fi
   else
    LANG=${LANGORG} echo "...skipped"
   fi
  fi
 done
fi

#create a list of all pkgs from the updates...
rm -f updatespkglist-* 2>/dev/null
UPDATEdbs="$(ls -1 Packages*updates* 2>/dev/null)"
if [ "$UPDATEdbs" ];then
 for aUPDATEdb in $UPDATEdbs
 do
  xaUPDATEdb="$(basename ${aUPDATEdb} pre)"
  aUDPtype="$(echo -n "$xaUPDATEdb" | cut -f 4- -d '-')" #ex: extract "main" from "Packages-devuan-ascii-main"
  UDPlist="$(grep -o '^Package: .*' ${aUPDATEdb} | tr -s ' ' | cut -f 2 -d ' ' | sort -u)"
  echo "$UDPlist" > updatespkglist-${aUDPtype}
 done
else
 PKGdbs="$(ls -1 Packages* 2>/dev/null)"
 for aPKGdb in $PKGdbs
 do
  xaPKGdb="$(basename ${aPKGdb} pre)"
  aPKGtype="$(echo -n "$xaPKGdb" | cut -f 4- -d '-')"
  echo -n '' > updatespkglist-${aPKGtype}
 done
fi

v1PATTERN="s%\\-[0-9.]*${DISTRO_BINARY_COMPAT}[0-9.]*$%%"
v2PATTERN="s%\\+[0-9.]*${DISTRO_BINARY_COMPAT}[0-9.]*$%%"

#debdb2pupdb.bac now also check that deps exist, need this list of pkgs...
if [ "$RUNNINGPUP" = "no" ];then #debdb2pupdb does not read this file in a running pup.
 echo -n " " > /tmp/0setupcompletelistpkgs
 COMPLETELISTPKGS="$(grep '^Package: ' Packages*pre 2>/dev/null | cut -f 2 -d ' ' | tr '\n' ' ')"
 echo "$COMPLETELISTPKGS" >> /tmp/0setupcompletelistpkgs
 [ -f /tmp/0setupnewinvaliddeps ] && rm -f /tmp/0setupnewinvaliddeps #see debdb2pupdb.
 echo -n '' > invaliddepslist
else
 [ -f /tmp/0setupcompletelistpkgs ] && rm -f /tmp/0setupcompletelistpkgs
 #debdb2pupdb expects /usr/local/petget/invaliddepslist...
 [ -f /usr/local/petget/invaliddepslist ] && mv -f /usr/local/petget/invaliddepslist /usr/local/petget/invaliddepslistHIDE
 cp -f /usr/local/EasyPak/invaliddepslist /usr/local/petget/invaliddepslist
fi

#convert pkg database ${PKGLISTFILE}pre to a standard format...
#each line: pkgname|nameonly|version|pkgrelease|category[;subcategory]|size|path|fullfilename|dependencies|description|
#ex: abiword-1.2.4|abiword|1.2.4|5|Document|999K|slackware/ab|abiword-1.2.4-5-i486.tgz|+aiksausus,+gtk2|a nice wordprocessor|
PKGLISTS_COMPAT="`echo "$PKG_DOCS_DISTRO_COMPAT" | tr ' ' '\n' | cut -f 3 -d '|' | tr '\n' ' '`" #see file DISTRO_PKGS_SPECS-ubuntu
#...format ex: 'Packages-ubuntu-intrepid-main Packages-ubuntu-intrepid-universe'
for ONE_PKGLISTS_COMPAT in $PKGLISTS_COMPAT $PKGLISTS_COMPAT_UPDATES
do
 [ ! -f ${ONE_PKGLISTS_COMPAT}pre ] && continue
 echo
 LANG=${LANGORG} echo "Processing ${ONE_PKGLISTS_COMPAT} into a standard format..."
 echo -n "" > $ONE_PKGLISTS_COMPAT
 echo -n "" > /tmp/${ONE_PKGLISTS_COMPAT}temp
 if [ -f /usr/local/petget/debdb2pupdb ];then
  #new fast compiled app...
  #create an intermediate-converted file...
  LANG=${LANGORG} echo " please wait..."
  MANIPULATED1="$(grep -E '^Package:|^Installed\-Size:|^Architecture:|^Version:|^Depends:|^Filename:|^Description:|^Homepage:|^Section:' ${ONE_PKGLISTS_COMPAT}pre | tr '[\t"|`]' ' ' | tr -s ' ' | sed -e 's%^Installed\-Size%InstalledSize%' -e 's%^Package:%STARTMARKER|PKG\nPackage:%' -e 's%: %|%' -e 's% $%%')" #`geany
  echo "$MANIPULATED1" > /tmp/woof-debdb.in
  #call new app. note, this in turn calls /usr/local/petget/find_pkgs...
  /usr/local/petget/debdb2pupdb ${DISTRO_BINARY_COMPAT} ${DISTRO_COMPAT_VERSION} > /tmp/${ONE_PKGLISTS_COMPAT}temp #130126 $ONE_PKGLISTS_COMPAT
  ${FIND_CAT} /tmp/${ONE_PKGLISTS_COMPAT}temp > $ONE_PKGLISTS_COMPAT
  #homepages also get logged...
  if [ -f /tmp/woof-homepages.acc ];then
   if [ -f ./PKGS_HOMEPAGES ];then #running in woof.
    cat  ./PKGS_HOMEPAGES >> /tmp/woof-homepages.acc
    sort --key=1,1 --field-separator=" " --unique /tmp/woof-homepages.acc > ./PKGS_HOMEPAGES
   fi
  fi
  #/tmp/0setupnewinvaliddeps is created by debdb2pupdb, when run in woof.
  # 3builddistro will have to copy this file 'invaliddepslist' into rootfs-complete/usr/local/petget/ ...
  # it is read by debdb2pupdb in a running puppy.
  if [ "$RUNNINGPUP" = "no" ];then
   NIDL1="$(sort -u /tmp/0setupnewinvaliddeps | tr '\n' ' ')"
   echo -n "$NIDL1" >> ./invaliddepslist
  fi
 else
  DB_category=""
  #debian squeeze and earlier has Description field last, wheezy has Filename field last...
  Description=''; Filename=''; Package=''; InstalledSize=''; Architecture=''; Version=''; Depends=''
  endflag=''
  while read DB_ONELINE
  do
   eval $DB_ONELINE
   case $DB_ONELINE in
    Description*)
    [ "$Filename" ] && endflag='yes'
    ;;
    Filename*)
     [ "$Description" ] && endflag='yes' #wheezy and later.
    ;;
   esac
   if [ "$endflag" = "yes" ];then
    DB_fullfilename="`echo -n "$Filename" | rev | cut -f 1 -d '/' | rev`"
    DB_path="`echo -n "$Filename" | rev | cut -f 2-9 -d '/' | rev`"
    DB_version="`echo -n "$Version" | sed -e 's%^[0-9]:%%' -e "$v1PATTERN" -e "$v2PATTERN"`"
    DB_pkgrelease="`echo -n "$Version" | sed -e 's%^[0-9]:%%' -e "s%${DB_version}%%" -e 's%^\\-%%' -e 's%^\\+%%'`"
    DB_pkgname="${Package}_${DB_version}"
    #for debian, ubuntu, use the generic name provided in the $DB_path, instead of $Package, for find_cat... 130126...
    #xPackage="$Package"
    #[ "$DB_path" ] && xPackage="$(basename "$DB_path")"
    #DB_category="`${FIND_CAT} $xPackage "$Description"`" #find_cat now returns category[;subcategory], ex: Document;edit
    DB_dependencies="`echo -n "$Depends" | sed -e 's%, %\n%g' | cut -f 1 -d ' ' | tr '\n' ' ' | sed -e 's% $%%' -e 's% %,+%g' -e 's%,$%%'`"
    [ "$DB_dependencies" != "" ] && DB_dependencies='+'"$DB_dependencies"
    echo "$DB_pkgname|$Package|$DB_version|$DB_pkgrelease|$DB_category|${InstalledSize}K|$DB_path|$DB_fullfilename|$DB_dependencies|$Description|$DISTRO_BINARY_COMPAT|$DISTRO_COMPAT_VERSION|" >> /tmp/${ONE_PKGLISTS_COMPAT}temp #130126 $ONE_PKGLISTS_COMPAT
    echo -n "$Package "
    Description=''; Filename=''; Package=''; InstalledSize=''; Architecture=''; Version=''; Depends='' #160411
    endflag=''
   fi
  done<<_END1
$(grep -E '^Package:|^Installed\-Size:|^Architecture:|^Version:|^Depends:|^Filename:|^Description:' ${ONE_PKGLISTS_COMPAT}pre | tr '[\t"|`]' ' ' | tr -s ' ' | grep -v '\\-dbg$' | sed -e 's%^Installed\-Size%InstalledSize%' -e 's%: %="%' -e 's% $%%' -e 's%$%"%')
_END1
  ${FIND_CAT} /tmp/${ONE_PKGLISTS_COMPAT}temp > $ONE_PKGLISTS_COMPAT
 fi
 #ubuntu trusty tahr, at 140215 have added lots of xorg pkgs with "-lts-" in name. don't see the point of them...
 sed -i -e '/\-lts\-/d' $ONE_PKGLISTS_COMPAT
 # -hwe- maybe should use these in preference too without, but for now just remove them
 #ref: https://wiki.ubuntu.com/Kernel/LTSEnablementStack
 sed -i -e '/\-hwe\-/d' $ONE_PKGLISTS_COMPAT
 echo
 #rm -f ${ONE_PKGLISTS_COMPAT}pre 2>/dev/null
 mv -f ${ONE_PKGLISTS_COMPAT}pre /tmp/ 2>/dev/null
 sort --field-separator='|' --key=1 ${ONE_PKGLISTS_COMPAT} > /tmp/$PKGLISTFILE
 mv -f /tmp/$PKGLISTFILE ${ONE_PKGLISTS_COMPAT}
done
echo

if [ "$RUNNINGPUP" == "yes" ];then
 if [ -f /usr/local/petget/invaliddepslistHIDE ];then
  mv -f /usr/local/petget/invaliddepslistHIDE /usr/local/petget/invaliddepslist
 else
  rm -f /usr/local/petget/invaliddepslist
 fi
else
 IDLu="$(cat invaliddepslist | tr -s ' ' | tr ' ' '\n' | sort -u | tr '\n' ' ')"
 echo "$IDLu" > invaliddepslist
fi

####################################
#merge updates into original db files...
if [ "$DBUPDATEFLAG" = "" ];then #empty now means yes.
 for ONE_PKGLISTS_COMPAT in $PKGLISTS_COMPAT
 do
  REPOFIELD="`echo -n "$ONE_PKGLISTS_COMPAT" | rev | cut -f 1 -d '-' | rev`" #ex: main
  PKGUPDATES="`echo -n "$PKGLISTS_COMPAT_UPDATES" | tr ' ' '\n' | grep "\-${REPOFIELD}$" | head -n 1`" #ex: Packages-ubuntu-precise_updates-main
  cat $PKGUPDATES > /tmp/0setup_xxx1
  cat $ONE_PKGLISTS_COMPAT >> /tmp/0setup_xxx1
  sync
  #want to discard the older package...
  sort --unique --field-separator='|' --key=2,2 /tmp/0setup_xxx1 > $ONE_PKGLISTS_COMPAT
  #...assumes pkg names remain the same, ex "firefox" (2nd field in db).
  mv -f $PKGUPDATES /tmp/$PKGUPDATES #dump -updates db file.
 done
fi
sync

###END###
