SickRage/lib/imdb/parser/sql/cutils.c

/*
 * cutils.c module.
 *
 * Miscellaneous functions to speed up the IMDbPY package.
 *
 * Contents:
 * - pyratcliff():
 *   Function that implements the Ratcliff-Obershelp comparison
 *   amongst Python strings.
 *
 * - pysoundex():
 *   Return a soundex code string, for the given string.
 *
 * Copyright 2004-2009 Davide Alberani <da@erlug.linux.it>
 * Released under the GPL license.
 *
 * NOTE: The Ratcliff-Obershelp part was heavily based on code from the
 * "simil" Python module.
 * The "simil" module is copyright of Luca Montecchiani <cbm64 _at_ inwind.it>
 * and can be found here: http://spazioinwind.libero.it/montecchiani/
 * It was released under the GPL license; original comments are leaved
 * below.
 *
 */


/*========== Ratcliff-Obershelp ==========*/
/*****************************************************************************
 *
 * Stolen code from :
 *
 * [Python-Dev] Why is soundex marked obsolete?
 * by Eric S. Raymond [4]esr@thyrsus.com
 * on Sun, 14 Jan 2001 14:09:01 -0500
 *
 *****************************************************************************/

/*****************************************************************************
 *
 * Ratcliff-Obershelp common-subpattern similarity.
 *
 * This code first appeared in a letter to the editor in Doctor
 * Dobbs's Journal, 11/1988.  The original article on the algorithm,
 * "Pattern Matching by Gestalt" by John Ratcliff, had appeared in the
 * July 1988 issue (#181) but the algorithm was presented in assembly.
 * The main drawback of the Ratcliff-Obershelp algorithm is the cost
 * of the pairwise comparisons.  It is significantly more expensive
 * than stemming, Hamming distance, soundex, and the like.
 *
 * Running time quadratic in the data size, memory usage constant.
 *
 *****************************************************************************/

#include <Python.h>

#define DONTCOMPARE_NULL    0.0
#define DONTCOMPARE_SAME    1.0
#define COMPARE             2.0
#define STRING_MAXLENDIFFER 0.7

/* As of 05 Mar 2008, the longest title is ~600 chars. */
#define MXLINELEN   1023

#define MAX(a,b) ((a) > (b) ? (a) : (b))


//*****************************************
// preliminary check....
//*****************************************
static float
strings_check(char const *s, char const *t)
{
    float threshold;    // lenght difference
    int s_len = strlen(s);    // length of s
    int t_len = strlen(t);    // length of t

    // NULL strings ?
    if ((t_len * s_len) == 0)
        return (DONTCOMPARE_NULL);

    // the same ?
    if (strcmp(s, t) == 0)
        return (DONTCOMPARE_SAME);

    // string lenght difference threshold
    // we don't want to compare too different lenght strings ;)
    if (s_len < t_len)
        threshold = (float) s_len / (float) t_len;
    else
        threshold = (float) t_len / (float) s_len;
    if (threshold < STRING_MAXLENDIFFER)
        return (DONTCOMPARE_NULL);

    // proceed
    return (COMPARE);
}


static int
RatcliffObershelp(char *st1, char *end1, char *st2, char *end2)
{
    register char *a1, *a2;
    char *b1, *b2;
    char *s1 = st1, *s2 = st2;    /* initializations are just to pacify GCC */
    short max, i;

    if (end1 <= st1 || end2 <= st2)
        return (0);
    if (end1 == st1 + 1 && end2 == st2 + 1)
        return (0);

    max = 0;
    b1 = end1;
    b2 = end2;

    for (a1 = st1; a1 < b1; a1++) {
        for (a2 = st2; a2 < b2; a2++) {
            if (*a1 == *a2) {
                /* determine length of common substring */
                for (i = 1; a1[i] && (a1[i] == a2[i]); i++)
                    continue;
                if (i > max) {
                    max = i;
                    s1 = a1;
                    s2 = a2;
                    b1 = end1 - max;
                    b2 = end2 - max;
                }
            }
        }
    }
    if (!max)
        return (0);
    max += RatcliffObershelp(s1 + max, end1, s2 + max, end2);    /* rhs */
    max += RatcliffObershelp(st1, s1, st2, s2);    /* lhs */
    return max;
}


static float
ratcliff(char *s1, char *s2)
/* compute Ratcliff-Obershelp similarity of two strings */
{
    int l1, l2;
    float res;

    // preliminary tests
    res = strings_check(s1, s2);
    if (res != COMPARE)
        return(res);

    l1 = strlen(s1);
    l2 = strlen(s2);

    return 2.0 * RatcliffObershelp(s1, s1 + l1, s2, s2 + l2) / (l1 + l2);
}


/* Change a string to lowercase. */
static void
strtolower(char *s1)
{
    int i;
    for (i=0; i < strlen(s1); i++) s1[i] = tolower(s1[i]);
}


/* Ratcliff-Obershelp for two python strings; returns a python float. */
static PyObject*
pyratcliff(PyObject *self, PyObject *pArgs)
{
    char *s1 = NULL;
    char *s2 = NULL;
    PyObject *discard = NULL;
    char s1copy[MXLINELEN+1];
    char s2copy[MXLINELEN+1];

    /* The optional PyObject parameter is here to be compatible
     * with the pure python implementation, which uses a
     * difflib.SequenceMatcher object. */
    if (!PyArg_ParseTuple(pArgs, "ss|O", &s1, &s2, &discard))
        return NULL;

    strncpy(s1copy, s1, MXLINELEN);
    strncpy(s2copy, s2, MXLINELEN);
    /* Work on copies. */
    strtolower(s1copy);
    strtolower(s2copy);

    return Py_BuildValue("f", ratcliff(s1copy, s2copy));
}


/*========== soundex ==========*/
/* Max length of the soundex code to output (an uppercase char and
 * _at most_ 4 digits). */
#define SOUNDEX_LEN 5

/* Group Number Lookup Table  */
static char soundTable[26] =
{ 0 /* A */, '1' /* B */, '2' /* C */, '3' /* D */, 0 /* E */, '1' /* F */,
 '2' /* G */, 0 /* H */, 0 /* I */, '2' /* J */, '2' /* K */, '4' /* L */,
 '5' /* M */, '5' /* N */, 0 /* O */, '1' /* P */, '2' /* Q */, '6' /* R */,
 '2' /* S */, '3' /* T */, 0 /* U */, '1' /* V */, 0 /* W */, '2' /* X */,
  0 /* Y */, '2' /* Z */};

static PyObject*
pysoundex(PyObject *self, PyObject *pArgs)
{
    int i, j, n;
    char *s = NULL;
    char word[MXLINELEN+1];
    char soundCode[SOUNDEX_LEN+1];
    char c;

    if (!PyArg_ParseTuple(pArgs, "s", &s))
        return NULL;

    j = 0;
    n = strlen(s);

    /* Convert to uppercase and exclude non-ascii chars. */
    for (i = 0; i < n; i++) {
        c = toupper(s[i]);
        if (c < 91 && c > 64) {
            word[j] = c;
            j++;
        }
    }
    word[j] = '\0';

    n = strlen(word);
    if (n == 0) {
        /* If the string is empty, returns None. */
        return Py_BuildValue("");
    }
    soundCode[0] = word[0];

    /* Build the soundCode string. */
    j = 1;
    for (i = 1; j < SOUNDEX_LEN && i < n; i++) {
        c = soundTable[(word[i]-65)];
        /* Compact zeroes and equal consecutive digits ("12234112"->"123412") */
        if (c != 0 && c != soundCode[j-1]) {
                soundCode[j++] = c;
        }
    }
    soundCode[j] = '\0';

    return Py_BuildValue("s", soundCode);
}


static PyMethodDef cutils_methods[] = {
    {"ratcliff", pyratcliff,
        METH_VARARGS, "Ratcliff-Obershelp similarity."},
    {"soundex", pysoundex,
        METH_VARARGS, "Soundex code for strings."},
    {NULL}
};


void
initcutils(void)
{
    Py_InitModule("cutils", cutils_methods);
}