#!/bin/bash #sourceFolder = /home/bigdatagfts/pl62716/refdata #targetFolder = /home/bigdatagfts/pl62716/refdata_target sourceFolder=$1 targetFolder=$2 if [ $# != 2 ] ; then echo "USAGE: $0 sourceFolder hdfsFolder" echo " e.g.: $0 /home/bigdatagfts/pl62716/refdata /home/bigdatagfts/pl62716/refdata_target" exit 1; fi if [ ! -d "$sourceFolder" ];then echo "$sourceFolder is not exist, please check!" exit 1; else if [ ! -d "$targetFolder" ];then echo "$targetFolder is not exist! create $targetFolder !" mkdir -p "$targetFolder" fi fi echo "delete lines which begin with H/T and store to dir $targetFolder" cd $sourceFolder for file in `ls` do if test -f $file then if [ -f "$targetFolder/$file" ]; then rm $targetFolder/$file fi sed '/^H\|^T/'d $file | cat -n > $targetFolder/$file fi done