//============================================================================ // // project: freedb // file: unicode-archive.c // author: Andrew Smith // date: 2005-01-01 // language: C // // NOTES // // FreeDB music CD database tarball conversion and analysis. // Detects whether a CDDB record is encoded as ISO-8859-1 or UTF-8. // // Copyright 2005, Andrew Smith // // //---------------------------------------------------------------------------- // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published // by the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // General Public License for more details. // // You should have received a copy of the GNU General Public License // along with this program; if not, write to the Free Software Foundation, // Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA // // //============================================================================ #include #include enum tTarSize { kMaxName = 100, kMaxMode = 8, kMaxUid = 8, kMaxGid = 8, kMaxSize = 12, kMaxTime = 12, kMaxChkSum = 8, kMaxTypeFlag = 1, kMaxLinkName = 100, kMaxMagic = 6, kMaxVersion = 2, kMaxUName = 32, kMaxGName = 32, kMaxDevMajor = 8, kMaxDevMinor = 8, kMaxPrefix = 155 }; enum tTarType { kARegType = 0, kRegType = '0', kLnkType = '1', kSymType = '2', kChrType = '3', kBlkType = '4', kDirType = '5', kFifoType = '6', kContType = '7' }; typedef struct { unsigned char fName[kMaxName]; unsigned char fMode[kMaxMode]; unsigned char fUid[kMaxUid]; unsigned char fGid[kMaxGid]; unsigned char fSize[kMaxSize]; unsigned char fTime[kMaxTime]; unsigned char fChkSum[kMaxChkSum]; unsigned char fTypeFlag; unsigned char fLinkName[kMaxLinkName]; unsigned char fMagic[kMaxMagic]; unsigned char fVersion[kMaxVersion]; unsigned char fUName[kMaxUName]; unsigned char fGName[kMaxGName]; unsigned char fDevMajor[kMaxDevMajor]; unsigned char fDevMinor[kMaxDevMinor]; unsigned char fPrefix[kMaxPrefix]; unsigned char fFiller[12]; } tTarRec; #define kBlockSize sizeof(tTarRec) static int gFileSize,gFileTime; static tTarRec gTar; //---------------------------------------------------------------------------- #define kMaxBuffer (4096 * kBlockSize) #define kFramesPerSecond 75 enum tBoolean {FALSE, TRUE}; enum tField { kLength, kRevision, kProcessor, kSubmitter, kDiscId, kDTitle, kDYear, kDGenre, kExtD, kFrame, kTTitle, kExtT, kMaxField }; enum tError { kLineError = 1, kFileError, kArchiveError, kLengthError, kFieldError, kBufferError, kCommentError, kCategoryError }; enum tTolerance {kNeeded, kEolOk, kEofOk}; enum tEncoding {kAscii7 = 'A', kLatin1 = 'L', kUtf8 = 'U'}; static unsigned char gBuffer[kMaxBuffer], gInput[kMaxBuffer]; static unsigned char *gNext, *gFrom, *gFields[kMaxField]; static int gField, gTrack, gCount = 0; static enum tEncoding gEncoding; //---------------------------------------------------------------------------- // handle an error static void HandleError(aError) enum tError aError; { if (aError) { unsigned char *vThis; for (vThis = gBuffer; vThis < gNext; ++vThis) putchar(*vThis); } exit((int)aError); } //---------------------------------------------------------------------------- // initialise variables static void Initialise(void) { for (gField = 0; gField < kMaxField; ++gField) gFields[gField] = NULL; gNext = gBuffer; gFrom = gInput; gTrack = -1; gCount++; } //---------------------------------------------------------------------------- // convert octal digits to an integer static int GetOctal(aDigitP) unsigned char *aDigitP; { int vValue = 0; while (*aDigitP >= '0' && *aDigitP <= '7') vValue = vValue * 8 + *aDigitP++ - '0'; return vValue; } //---------------------------------------------------------------------------- // convert decimal digits to an integer static int GetDecimal(aDigitP) unsigned char *aDigitP; { int vValue = 0; while (*aDigitP >= '0' && *aDigitP <= '9') vValue = vValue * 10 + *aDigitP++ - '0'; return vValue; } //---------------------------------------------------------------------------- // convert hexadecimal digits to an integer static unsigned GetHex(aDigitP) unsigned char *aDigitP; { unsigned vValue = 0; while (TRUE) if (*aDigitP >= '0' && *aDigitP <= '9') vValue = vValue * 16 + *aDigitP++ - '0'; else if (*aDigitP >= 'a' && *aDigitP <= 'f') vValue = vValue * 16 + *aDigitP++ - 'a' + 10; else return vValue; } //---------------------------------------------------------------------------- // map category code to a letter of the alphabet static unsigned char GetCategory(aDirP) unsigned char *aDirP; { if (*aDirP == '.') aDirP += 2; switch (*aDirP) { case 'b': // blues=A return 'A'; case 'c': // classical=B, country=C return *(aDirP+1) == 'l' ? 'B' : 'C'; case 'd': // data=D return 'D'; case 'f': // folk=E return 'E'; case 'j': // jazz=F return 'F'; case 'm': // misc=G return 'G'; case 'n': // newage=H return 'H'; case 'r': // reggae=I, rock=J return *(aDirP+1) == 'e' ? 'I' : 'J'; case 's': // soundtrack=K return 'K'; default: HandleError(kCategoryError); return; } } //---------------------------------------------------------------------------- // extract a disc id from a file path static int gCatLength[] = {6,10,8,5,5,5,5,7,7,5,11}; static unsigned GetDiscId(aDirP,aCategory) unsigned char *aDirP; unsigned char aCategory; { if (*aDirP == '.') aDirP += 2; return GetHex(aDirP + gCatLength[aCategory - 'A']); } //---------------------------------------------------------------------------- // check for a string containing the given track number static enum tBoolean NumberedText(aTextP,aTrack) unsigned char *aTextP; int aTrack; { unsigned char vDigit1 = 0, vDigit2 = 0; if (aTrack >= 10) { vDigit2 = '0' + aTrack % 10; vDigit1 = '0' + aTrack / 10; } else if (aTrack >= 1) vDigit1 = '0' + aTrack; for (; *aTextP; ++aTextP) if (*aTextP == vDigit1 && (vDigit2 == 0 || *(aTextP+1) == vDigit2)) return TRUE; return FALSE; } //---------------------------------------------------------------------------- // return the length of the next character or zero at end of string // assumes that text is encoded as UTF-8 static int CharLength(aTextP) unsigned char *aTextP; { return *aTextP == '\\' || *aTextP >= 0x80 ? 2 : *aTextP > 0 ? 1 : 0; } //---------------------------------------------------------------------------- // test if the next character represents white space static int WhiteSpace(aTextP) unsigned char *aTextP; { return *aTextP > 0 && *aTextP <= 0x20 || *aTextP == '\\' && (*aTextP == 't' || *aTextP == 'r' || *aTextP == 'n'); } //---------------------------------------------------------------------------- // remove leading and trailing white space and replace runs of white space // with a single space static int CleanText(aTextP,aDirtyF) unsigned char *aTextP; int aDirtyF; { int vLength; unsigned char *vNextP,*vLastP; if (aTextP == NULL || *aTextP == 0) return aDirtyF; vLength = CharLength(aTextP); vNextP = vLastP = aTextP; while (*vNextP) { for (; *vNextP && WhiteSpace(vNextP); vLength = CharLength(vNextP)) vNextP += vLength; if (*vNextP && vLastP > aTextP) { if (*vLastP != ' ') { aDirtyF = TRUE; *vLastP = ' '; } vLastP++; } for (; *vNextP && !WhiteSpace(vNextP); vLength = CharLength(vNextP)) switch (vLength) { case 2: *vLastP++ = *vNextP++; case 1: *vLastP++ = *vNextP++; break; } } if (*vLastP) { aDirtyF = TRUE; *vLastP = 0; } return aDirtyF; } //---------------------------------------------------------------------------- // determine the encoding of a single field, given the record encoding static enum tEncoding FieldEncoding(aTextP,aEncoding) unsigned char *aTextP; enum tEncoding aEncoding; { enum tEncoding vEncoding; while (*aTextP) if (*aTextP++ >= 0x80) return aEncoding; return kAscii7; } //---------------------------------------------------------------------------- // output disc and track records // disc records have an empty track field // track records begin with a digit // artist and title fields are prefixed with a one letter encoding indicator static void OutputDisc(void) { unsigned char vCategory = GetCategory(gTar.fName); unsigned char *vArtist, *vTitle, *vExt, *vYear, *vGenre; unsigned char *vSep, vVarious = 'f'; unsigned char vArtistEncoding, vTitleEncoding; unsigned char vExtEncoding, vGenreEncoding; unsigned vDiscId = GetDiscId(gTar.fName,vCategory); int vTrack, vThisFrame = 0, vNextFrame = -1; int vDirtyF = FALSE, vNumberedF = TRUE; // get the first frame offset if (gFields[kFrame]) { vThisFrame = GetDecimal(gFields[kFrame]); while (*gFields[kFrame]++); } for (vTrack = 0; vTrack <= gTrack; ++vTrack) { // separate artist and title fields vArtist = vTitle = (unsigned char*) ""; if (gFields[kTTitle]) { for (vSep = gFields[kTTitle]; *vSep; ++vSep) if (*vSep == ' ' && *(vSep + 1) == '/' && *(vSep + 2) == ' ') break; if (*vSep) { vVarious = 't'; // note the presence of track artists vArtist = gFields[kTTitle]; vTitle = vSep + 3; gFields[kTTitle] = vTitle; *vSep = 0; } else vTitle = gFields[kTTitle]; } // get the next frame offset if (gFields[kFrame] && vTrack < gTrack) { vNextFrame = GetDecimal(gFields[kFrame]); while (*gFields[kFrame]++); } else { gFields[kFrame] = NULL; if (gFields[kLength]) vNextFrame = GetDecimal(gFields[kLength]) * kFramesPerSecond; else vNextFrame = 0; } // check whether fields will need to be cleaned vExt = gFields[kExtT]; vNumberedF = vNumberedF && (NumberedText(vArtist,vTrack+1) || NumberedText(vTitle,vTrack+1)); if (gFields[kTTitle]) while (*gFields[kTTitle]++); if (gFields[kExtT]) while (*gFields[kExtT]++); vDirtyF = CleanText(vArtist,vDirtyF); vDirtyF = CleanText(vTitle,vDirtyF); vDirtyF = CleanText(vExt,vDirtyF); vArtistEncoding = FieldEncoding(vArtist,gEncoding); vTitleEncoding = FieldEncoding(vTitle,gEncoding); vExtEncoding = FieldEncoding(vExt,gEncoding); // output a track record printf("%d\t%d\t%d\t%d\t%c%s\t%c%s\t%c%s\n", vTrack, gCount, vThisFrame, vNextFrame - vThisFrame, vArtistEncoding, vArtist, vTitleEncoding, vTitle, vExtEncoding, vExt); vThisFrame = vNextFrame; } // separate artist and title fields in disc records vArtist = vTitle = (unsigned char*) ""; if (gFields[kDTitle]) { for (vSep = gFields[kDTitle]; *vSep; ++vSep) if (*vSep == ' ' && *(vSep + 1) == '/' && *(vSep + 2) == ' ') break; if (*vSep) { vArtist = gFields[kDTitle]; vTitle = vSep + 3; *vSep = 0; } else vTitle = gFields[kDTitle]; } // check whether fields will need to be cleaned vExt = gFields[kExtD] ? gFields[kExtD] : (unsigned char*) ""; vYear = gFields[kDYear] ? gFields[kDYear] : (unsigned char*) ""; vGenre = gFields[kDGenre] ? gFields[kDGenre] : (unsigned char*) ""; vDirtyF = CleanText(gFields[kLength],vDirtyF); vDirtyF = CleanText(gFields[kRevision],vDirtyF); vDirtyF = CleanText(gFields[kProcessor],vDirtyF); vDirtyF = CleanText(gFields[kSubmitter],vDirtyF); vDirtyF = CleanText(gFields[kDiscId],vDirtyF); vDirtyF = CleanText(vArtist,vDirtyF); vDirtyF = CleanText(vTitle,vDirtyF); vDirtyF = CleanText(vYear,vDirtyF); vDirtyF = CleanText(vGenre,vDirtyF); vDirtyF = CleanText(vExt,vDirtyF); vArtistEncoding = FieldEncoding(vArtist,gEncoding); vTitleEncoding = FieldEncoding(vTitle,gEncoding); vExtEncoding = FieldEncoding(vExt,gEncoding); vGenreEncoding = FieldEncoding(vGenre,gEncoding); // output a disc record printf("\t%d\t%s\t%s\t%s\t%s\t%s\t%c%s\t%c%s\t%s" "\t%c%s\t%c%s\t%c\t%d\t%d\t%d\t%d\t%c\t%c" "\t%c\t%c\n", gCount, gFields[kLength] ? gFields[kLength] : (unsigned char*) "", gFields[kRevision] ? gFields[kRevision] : (unsigned char*) "0", gFields[kProcessor] ? gFields[kProcessor] : (unsigned char*) "", gFields[kSubmitter] ? gFields[kSubmitter] : (unsigned char*) "", gFields[kDiscId] ? gFields[kDiscId] : (unsigned char*) "", vArtistEncoding, vArtist, vTitleEncoding, vTitle, vYear, vGenreEncoding, vGenre, vExtEncoding, vExt, vCategory, vDiscId, gFileTime, gFileSize, gTrack + 1, vVarious, vDirtyF ? 't' : 'f', gEncoding, vNumberedF ? 't' : 'f'); } //---------------------------------------------------------------------------- // output a link record // link records begin with a letter static void OutputLink(void) { unsigned char vFileCat = GetCategory(gTar.fName); unsigned char vLinkCat = GetCategory(gTar.fLinkName); unsigned vFileId = GetDiscId(gTar.fName,vFileCat); unsigned vLinkId = GetDiscId(gTar.fLinkName,vLinkCat); printf("%c\t%d\t%d\t%c\t%d\n", vFileCat, vFileId, gFileTime, vLinkCat, vLinkId); } //---------------------------------------------------------------------------- // retrieve the next character static int GetNext(aTolerance) enum tTolerance aTolerance; { int vChar = *gFrom++; switch (vChar) { case 0: HandleError(kFileError); case '\n': if (aTolerance < kEolOk) HandleError(kLineError); default: *gNext = vChar; break; } return vChar; } //---------------------------------------------------------------------------- // commit a character, escaping or converting special characters static void PutNext(void) { static int vSlashF = FALSE; if (gNext >= gBuffer + kMaxBuffer - 3) { OutputDisc(); HandleError(kBufferError); } // put out a pending back slash if (vSlashF) { vSlashF = FALSE; switch (*gNext) { case 't': case 'r': case 'n': case '\\': ++gNext; return; default: *(gNext+1) = *gNext; *gNext++ = '\\'; break; } } // escape tabs, returns and backslashes // null terminate lines // convert ISO-8859-1 to UTF-8 switch (*gNext) { case '\n': *gNext++ = 0; break; case '\t': *gNext++ = '\\'; *gNext++ = 't'; break; case '\r': *gNext++ = '\\'; *gNext++ = 'r'; break; case '\\': vSlashF = TRUE; default: if (*gNext >= 0x80 && gEncoding == kLatin1) { *(gNext+1) = (0x80 | (*gNext & 0x3F)); *gNext++ = (0xC0 | ((*gNext >> 6) & 0x1F)); } ++gNext; break; } } //---------------------------------------------------------------------------- // save a line, escaping special characters static void CopyLine(void) { do PutNext(); while (GetNext(kEolOk) != '\n'); PutNext(); } //---------------------------------------------------------------------------- // discard a line static void SkipLine(void) { while (GetNext(kEolOk) != '\n'); } //---------------------------------------------------------------------------- // save a field static void CopyField(void) { if (*gNext == '\n') PutNext(); else CopyLine(); } //---------------------------------------------------------------------------- // skip a disc field and output the rest of the line static void SkipDisc(aField) enum tField aField; { while (GetNext(kNeeded) != '='); if (aField == gField) --gNext; else { gFields[aField] = gNext; gField = aField; } GetNext(kEolOk); CopyField(); } //---------------------------------------------------------------------------- // skip a track field and output the rest of the line static void SkipTrack(aField) enum tField aField; { int vTrack = 0; while (GetNext(kNeeded), *gNext < '0' || *gNext > '9'); do vTrack = vTrack * 10 + (*gNext - '0'); while (GetNext(kNeeded), *gNext >= '0' && *gNext <= '9'); if (*gNext != '=') HandleError(kFieldError); if (aField == gField && vTrack == gTrack) --gNext; else { if (aField != gField) { gFields[aField] = gNext; gField = aField; } gTrack = vTrack; } GetNext(kEolOk); CopyField(); } //---------------------------------------------------------------------------- // skip past a colon and white space*/ static enum tBoolean SkipColon(aField) enum tField aField; { while (GetNext(kEolOk), *gNext != ':' && *gNext != '\n'); if (*gNext == '\n') return FALSE; while (GetNext(kEolOk), *gNext == ' ' || *gNext == '\t'); gFields[aField] = gNext; gField = aField; return TRUE; } //---------------------------------------------------------------------------- // send only a number static void CopyNumber(void) { do PutNext(); while (GetNext(kEolOk), *gNext >= '0' && *gNext <= '9'); while (*gNext != '\n') GetNext(kEolOk); PutNext(); } //---------------------------------------------------------------------------- // skip white space and analyse a comment line static void SkipComment(void) { while (GetNext(kEolOk), *gNext == ' ' || *gNext == '\t'); switch (*gNext) { case 'D': //# Disc length: if (SkipColon(kLength)) CopyNumber(); break; case 'R': //# Revision: if (SkipColon(kRevision)) CopyNumber(); break; case 'P': //# Processed by: if (SkipColon(kProcessor)) CopyField(); break; case 'S': //# Submitted via: if (SkipColon(kSubmitter)) CopyField(); break; case '\n': break; default: //# [0-9]+ if (*gNext >= '0' && *gNext <= '9') { if (gField != kFrame) { gFields[kFrame] = gNext; gField = kFrame; } CopyNumber(); } else SkipLine(); break; } } //---------------------------------------------------------------------------- // process a cddb file static void CopyFile(void) { while (TRUE) switch (GetNext(kEofOk)) { case '#': SkipComment(); break; case 'D': switch (GetNext(kNeeded)) { case 'I': //DISCID= SkipDisc(kDiscId); break; case 'T': //DTITLE[0-9]+= SkipDisc(kDTitle); break; case 'Y': //DYEAR= SkipDisc(kDYear); break; case 'G': //DGENRE= SkipDisc(kDGenre); break; default: HandleError(kFieldError); } break; case 'T': //TTITLE[0-9]+= SkipTrack(kTTitle); break; case 'E': GetNext(kNeeded); GetNext(kNeeded); switch (GetNext(kNeeded)) { case 'D': //EXTD= SkipDisc(kExtD); break; case 'T': //EXTT= SkipTrack(kExtT); break; default: HandleError(kFieldError); } break; case 'P': //PLAYORDER= OutputDisc(); return; default: SkipLine(); break; } } //---------------------------------------------------------------------------- // read bytes into a buffer static void GetBlocks(aTolerance,aBufferP,aBlocks) enum tTolerance aTolerance; unsigned char *aBufferP; size_t aBlocks; { if (fread(aBufferP,kBlockSize,aBlocks,stdin) < aBlocks) HandleError(aTolerance < kEofOk || ferror(stdin) ? kLengthError : 0); } //---------------------------------------------------------------------------- // check to see if a file is encoded as UTF-8 (TRUE) or ISO-8859-1 (FALSE) // legitimate utf-8 sequences are: // // 0x00000000 - 0x0000007F: 0xxxxxxx // 0x00000080 - 0x000007FF: 110xxxxx 10xxxxxx // 0x00000800 - 0x0000FFFF: 1110xxxx 10xxxxxx 10xxxxxx // 0x00010000 - 0x001FFFFF: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx // 0x00200000 - 0x03FFFFFF: 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx // 0x04000000 - 0x7FFFFFFF: 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx // // The presence of any other bytes greater than or equal to 0x80 implies // that the string is encoded as ISO-8859-1. // static enum tEncoding TextEncoding(aTextP) unsigned char *aTextP; { int vLength; enum tEncoding vEncoding = kAscii7; for (; *aTextP; ++aTextP) { if ((*aTextP & 0xFE) == 0xFC) vLength = 5; else if ((*aTextP & 0xFC) == 0xF8) vLength = 4; else if ((*aTextP & 0xF8) == 0xF0) vLength = 3; else if ((*aTextP & 0xF0) == 0xE0) vLength = 2; else if ((*aTextP & 0xE0) == 0xC0) vLength = 1; else if ((*aTextP & 0x80) == 0x00) vLength = 0; else return kLatin1; if (vLength > 0) vEncoding = kUtf8; while (0 < vLength--) if ((*++aTextP & 0xC0) != 0x80) return kLatin1; } return vEncoding; } //---------------------------------------------------------------------------- // read a file header and contents from a tar archive on stdin static void GetFile(void) { Initialise(); GetBlocks(kEofOk,&gTar,1); gFileSize = GetOctal(gTar.fSize); gFileTime = GetOctal(gTar.fTime); switch (gTar.fTypeFlag) { case kARegType: case kRegType: case kContType: if (gFileSize >= kMaxBuffer) HandleError(kBufferError); if (gFileSize) { size_t vBlocks = (gFileSize + kBlockSize - 1) / kBlockSize; GetBlocks(kNeeded,gInput,vBlocks); gInput[gFileSize] = 0; if (GetNext(kEofOk) == '#' && GetNext(kEofOk) == ' ' && GetNext(kEofOk) == 'x' && GetNext(kEofOk) == 'm' && GetNext(kEofOk) == 'c' && GetNext(kEofOk) == 'd') { gEncoding = TextEncoding(gInput); SkipLine(); CopyFile(); } } break; case kLnkType: case kSymType: OutputLink(); break; case kDirType: break; case kChrType: case kBlkType: case kFifoType: default: fprintf(stderr,"bad archive type %s %x\n",gTar.fName,gTar.fTypeFlag); HandleError(kArchiveError); break; } } //---------------------------------------------------------------------------- // search for the start of cddb file marker extern int main(void) { while (TRUE) GetFile(); } //============================================================================