parselo.cpp

Go to the documentation of this file.

/*
 * Copyright (C) Volition, Inc. 1999.  All rights reserved.
 *
 * All source code herein is the property of Volition, Inc. You may not sell
 * or otherwise commercially exploit the source or things you created based on the
 * source.
 *
*/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdarg.h>
#include <setjmp.h>

#include "ctype.h"
#include "globalincs/version.h"
#include "localization/fhash.h"
#include "localization/localize.h"
#include "mission/missionparse.h"
#include "parse/encrypt.h"
#include "parse/parselo.h"
#include "parse/sexp.h"
#include "ship/ship.h"
#include "weapon/weapon.h"



#define ERROR_LENGTH    64
#define RS_MAX_TRIES    5

// to know that a modular table is currently being parsed
bool    Parsing_modular_table = false;

char            Current_filename[MAX_PATH_LEN];
char            Current_filename_save[MAX_PATH_LEN];
char            Current_filename_sub[MAX_PATH_LEN];     //Last attempted file to load, don't know if ex or not.
char            Error_str[ERROR_LENGTH];
int             my_errno;
int             Warning_count, Error_count;
int             Warning_count_save = 0, Error_count_save = 0;
int             fred_parse_flag = 0;
int             Token_found_flag;

char    *Mission_text = NULL;
char    *Mission_text_raw = NULL;
char    *Mp = NULL, *Mp_save = NULL;
const char      *token_found;

static int Parsing_paused = 0;

// text allocation stuff
void allocate_mission_text(int size);
static int Mission_text_size = 0;


//      Return true if this character is white space, else false.
int is_white_space(char ch)
{
        return ((ch == ' ') || (ch == '\t') || (ch == EOLN));
}

// Returns true if this character is gray space, else false (gray space is white space except for EOLN).
int is_gray_space(char ch)
{
        return ((ch == ' ') || (ch == '\t'));
}

int is_parenthesis(char ch)
{
        return ((ch == '(') || (ch == ')'));
}

//      Advance global Mp (mission pointer) past all current white space.
//      Leaves Mp pointing at first non white space character.
void ignore_white_space()
{
        while ((*Mp != EOF_CHAR) && is_white_space(*Mp))
                Mp++;
}

void ignore_gray_space()
{
        while ((*Mp != EOF_CHAR) && is_gray_space(*Mp))
                Mp++;
}

//      Truncate *str, eliminating all trailing white space.
//      Eg: "abc   "   becomes "abc"
//               "abc abc " becomes "abc abc"
//               "abc \t"   becomes "abc"
void drop_trailing_white_space(char *str)
{
        int     i = strlen(str) - 1;

        while ((i >= 0) && is_white_space(str[i]))
                i--;

        str[i+1] = 0;
}

//      Ditto for SCP_string
void drop_trailing_white_space(SCP_string &str)
{
        int i = str.length() - 1;

        while ((i >= 0) && is_white_space(str[i]))
                i--;

        str.resize(i+1);
}

//      Eliminate any leading whitespace in str
void drop_leading_white_space(char *str)
{
        int len, first;

        len = strlen(str);
        first = 0;

        // find first non-whitespace
        while ((first < len) && is_white_space(str[first]))
                first++;

        // quick out
        if (first == 0)
                return;

        memmove(str, str+first, len-first);
        str[len-first] = 0;
}

//      Ditto for SCP_string
void drop_leading_white_space(SCP_string &str)
{
        int len, first, i;

        len = str.length();
        first = 0;

        // find first non-whitespace
        while ((first < len) && is_white_space(str[first]))
                first++;

        // quick out
        if (first == 0)
                return;

        // copy chars to beginning of string
        for (i = 0; (first + i) < len; i++)
                str[i] = str[first + i];

        // since i is now off the end of the for loop, it represents the new length
        str.resize(i);
}

// eliminates all leading and trailing white space from a string.  Returns pointer passed in.
char *drop_white_space(char *str)
{
        int s, e;

        s = 0;
        while (str[s] && is_white_space(str[s]))
                s++;

        e = strlen(str) - 1;
        while (e > s) {
                if (!is_white_space(str[e]))
                        break;

                e--;
        }

        if (e > s)
                memmove(str, str + s, e - s + 1);

        str[e - s + 1] = 0;
        return str;
}

// ditto for SCP_string
void drop_white_space(SCP_string &str)
{
        int len, newlen, first, last, i;

        len = str.length();
        first = 0;
        last = len - 1;

        // find first non-whitespace
        while ((first < len) && is_white_space(str[first]))
                first++;

        // find last non-whitespace
        while ((last > first) && is_white_space(str[last]))
                last--;

        newlen = last - first + 1;

        // quick out
        if (newlen <= 0)
        {
                str = "";
                return;
        }

        if (first != 0)
        {
                // copy chars to beginning of string
                for (i = 0; i < newlen; i++)
                        str[i] = str[first + i];
        }

        str.resize(newlen);
}

//      Advances Mp past current token.
void skip_token()
{
        ignore_white_space();

        while ((*Mp != EOF_CHAR) && !is_white_space(*Mp))
                Mp++;
}

//      Display a diagnostic message if Verbose is set.
//      (Verbose is set if -v command line switch is present.)
void diag_printf(char *format, ...)
{
#ifndef NDEBUG
        SCP_string buffer;
        va_list args;

        va_start(args, format);
        vsprintf(buffer, format, args);
        va_end(args);

        nprintf(("Parse", "%s", buffer.c_str()));
#endif
}

//      Grab and return (a pointer to) a bunch of tokens, terminating at
// ERROR_LENGTH chars, or end of line.
char *next_tokens()
{
        int     count = 0;
        char    *pstr = Mp;
        char    ch;

        while (((ch = *pstr++) != EOLN) && (ch != EOF_CHAR) && (count < ERROR_LENGTH-1))
                Error_str[count++] = ch;

        Error_str[count] = 0;
        return Error_str;
}

//      Return the line number given by the current mission pointer, ie Mp.
//      A very slow function (scans all processed text), but who cares how long
//      an error reporting function takes?
int get_line_num()
{
        int     count = 1;
        int     incomment = 0;
        int     multiline = 0;
        char    *stoploc;
        char    *p;

        p = Mission_text;
        stoploc = Mp;

        while (p < stoploc)
        {
                if (*p == EOF_CHAR)
                        Assert(0);

                if ( !incomment && (*p == COMMENT_CHAR) )
                        incomment = 1;

                if ( !incomment && (*p == '/') && (*(p+1) == '*') ) {
                        multiline = 1;
                        incomment = 1;
                }

                if ( incomment )
                        stoploc++;

                if ( multiline && (*(p-1) == '*') && (*p == '/') ) {
                        multiline = 0;
                        incomment = 0;
                }

                if (*p++ == EOLN) {
                        if ( !multiline && incomment )
                                incomment = 0;
                        count++;
                }
        }

        return count;
}

//      Call this function to display an error message.
//      error_level == 0 means this is just a warning.
//      !0 means it's an error message.
//      Prints line number and other useful information.
extern int Cmdline_noparseerrors;
void error_display(int error_level, char *format, ...)
{
        char type[8];
        SCP_string error_text;
        va_list args;

        if (error_level == 0) {
                strcpy_s(type, "Warning");
                Warning_count++;
        } else {
                strcpy_s(type, "Error");
                Error_count++;
        }

        va_start(args, format);
        vsprintf(error_text, format, args);
        va_end(args);

        nprintf((type, "%s(line %i): %s: %s\n", Current_filename, get_line_num(), type, error_text.c_str()));

        if(error_level == 0 || Cmdline_noparseerrors)
                Warning(LOCATION, "%s(line %i):\n%s: %s", Current_filename, get_line_num(), type, error_text.c_str());
        else
                Error(LOCATION, "%s(line %i):\n%s: %s", Current_filename, get_line_num(), type, error_text.c_str());
}

//      Advance Mp to the next eoln character.
void advance_to_eoln(char *more_terminators)
{
        char    terminators[128];

        Assert((more_terminators == NULL) || (strlen(more_terminators) < 125));

        terminators[0] = EOLN;
        terminators[1] = EOF_CHAR;
        terminators[2] = 0;
        if (more_terminators != NULL)
                strcat_s(terminators, more_terminators);
        else
                terminators[2] = 0;

        while (strchr(terminators, *Mp) == NULL)
                Mp++;
}

// Advance Mp to the next white space (ignoring white space inside of " marks)
void advance_to_next_white()
{
        int in_quotes = 0;

        while ((*Mp != EOLN) && (*Mp != EOF_CHAR)) {
                if (*Mp == '\"')
                        in_quotes = !in_quotes;

                if (!in_quotes && is_white_space(*Mp))
                        break;

                if (!in_quotes && is_parenthesis(*Mp))
                        break;

                Mp++;
        }
}

// Search for specified string, skipping everything up to that point.  Returns 1 if found,
// 0 if string wasn't found (and hit end of file), or -1 if not found, but end of checking
// block was reached.
int skip_to_string(char *pstr, char *end)
{
        int len, len2 = 0;

        ignore_white_space();
        len = strlen(pstr);
        if (end)
                len2 = strlen(end);

        while ((*Mp != EOF_CHAR) && strnicmp(pstr, Mp, len)) {
                if (end && *Mp == '#')
                        return 0;

                if (end && !strnicmp(end, Mp, len2))
                        return -1;

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (!Mp || (*Mp == EOF_CHAR))
                return 0;

        Mp += strlen(pstr);
        return 1;
}

// Goober5000
// Advance to start of pstr.  Return 0 is successful, otherwise return !0
int skip_to_start_of_string(char *pstr, char *end)
{
        int len, endlen;

        ignore_white_space();
        len = strlen(pstr);
        if(end)
                endlen = strlen(end);
        else
                endlen = 0;

        while ( (*Mp != EOF_CHAR) && strnicmp(pstr, Mp, len) ) {
                if (end && *Mp == '#')
                        return 0;

                if (end && !strnicmp(end, Mp, endlen))
                        return 0;

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (!Mp || (*Mp == EOF_CHAR))
                return 0;

        return 1;
}

// Advance to start of either pstr1 or pstr2.  Return 0 is successful, otherwise return !0
int skip_to_start_of_string_either(char *pstr1, char *pstr2, char *end)
{
        int len1, len2, endlen;

        ignore_white_space();
        len1 = strlen(pstr1);
        len2 = strlen(pstr2);
        if(end)
                endlen = strlen(end);
        else
                endlen = 0;

        while ( (*Mp != EOF_CHAR) && strnicmp(pstr1, Mp, len1) && strnicmp(pstr2, Mp, len2) ) {
                if (end && *Mp == '#')
                        return 0;

                if (end && !strnicmp(end, Mp, endlen))
                        return 0;

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (!Mp || (*Mp == EOF_CHAR))
                return 0;

        return 1;
}

// Find a required string.
// If not found, display an error message, but try up to RS_MAX_TRIES times
// to find the string.  (This is the groundwork for ignoring non-understood
// lines.
//      If unable to find the required string after RS_MAX_TRIES tries, then
//      abort using longjmp to parse_abort.
int required_string(const char *pstr)
{
        int     count = 0;

        ignore_white_space();

        while (strnicmp(pstr, Mp, strlen(pstr)) && (count < RS_MAX_TRIES)) {
                error_display(1, "Missing required token: [%s]. Found [%.32s] instead.\n", pstr, next_tokens());
                advance_to_eoln(NULL);
                ignore_white_space();
                count++;
        }

        if (count == RS_MAX_TRIES) {
                nprintf(("Error", "Error: Unable to find required token [%s]\n", pstr));
                Warning(LOCATION, "Error: Unable to find required token [%s]\n", pstr);
        throw parse::ParseException("Required string not found");
        }

        Mp += strlen(pstr);
        diag_printf("Found required string [%s]\n", token_found = pstr);
        return 1;
}

int check_for_eof()
{
        ignore_white_space();

        if (*Mp == EOF_CHAR)
                return 1;

        return 0;
}

int check_for_eoln()
{
        ignore_gray_space();

        if(*Mp == EOLN)
                return 1;
        else
                return 0;
}

// similar to optional_string, but just checks if next token is a match.
// It doesn't advance Mp except to skip past white space.
int check_for_string(const char *pstr)
{
        ignore_white_space();

        if (!strnicmp(pstr, Mp, strlen(pstr)))
                return 1;

        return 0;
}

// like check for string, but doesn't skip past any whitespace
int check_for_string_raw(const char *pstr)
{
        if (!strnicmp(pstr, Mp, strlen(pstr)))
                return 1;

        return 0;
}

// Find an optional string.
//      If found, return 1, else return 0.
//      If found, point past string, else don't update pointer.
int optional_string(const char *pstr)
{
        ignore_white_space();

        if (!strnicmp(pstr, Mp, strlen(pstr))) {
                Mp += strlen(pstr);
                return 1;
        }

        return 0;
}

int optional_string_either(char *str1, char *str2)
{
        ignore_white_space();

        if ( !strnicmp(str1, Mp, strlen(str1)) ) {
                Mp += strlen(str1);
                return 0;
        } else if ( !strnicmp(str2, Mp, strlen(str2)) ) {
                Mp += strlen(str2);
                return 1;
        }

        return -1;
}

// generic parallel to required_string_one_of
int optional_string_one_of(int arg_count, ...)
{
        Assertion(arg_count > 0, "optional_string_one_of() called with arg_count of %d; get a coder!\n", arg_count);
        int idx, found = -1;
        char *pstr;
        va_list vl;

        ignore_white_space();

        va_start(vl, arg_count);
        for (idx = 0; idx < arg_count; idx++)
        {
                pstr = va_arg(vl, char*);

                if ( !strnicmp(pstr, Mp, strlen(pstr)) )
                {
                        Mp += strlen(pstr);
                        found = idx;
                        break;
                }
        }
        va_end(vl);

        return found;
}

int required_string_fred(char *pstr, char *end)
{
        char *backup = Mp;

        token_found = pstr;
        if (fred_parse_flag)
                return 0;

        ignore_white_space();
        while (*Mp != EOF_CHAR && strnicmp(pstr, Mp, strlen(pstr))) {
                if ((*Mp == '#') || (end && !strnicmp(end, Mp, strlen(end)))) {
                        Mp = NULL;
                        break;
                }

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (!Mp || (*Mp == EOF_CHAR)) {
                diag_printf("Required string [%s] not found\n", pstr);
                Mp = backup;
                Token_found_flag = 0;
                return 0;
        }

        Mp += strlen(pstr);
        diag_printf("Found required string [%s]\n", pstr);
        Token_found_flag = 1;
        return 1;
}

// attempt to find token in buffer.  It might not exist, however, in which case we don't need
// to do anything.  If it is found, then we advance the pointer to just after the token.  To
// further complicate things, we should only search to a certain point, since we don't want
// a token that belongs to another section which might match the token we want.  Thus, we
// also pass in an ending token, which marks the point we should stop looking at.
int optional_string_fred(char *pstr, char *end, char *end2)
{
        char *mp_save = Mp;

        token_found = pstr;
        if (fred_parse_flag)
                return 0;

        ignore_white_space();
        while ((*Mp != EOF_CHAR) && strnicmp(pstr, Mp, strlen(pstr))) {
                if ((*Mp == '#') || (end && !strnicmp(end, Mp, strlen(end))) ||
                        (end2 && !strnicmp(end2, Mp, strlen(end2)))) {
                        Mp = NULL;
                        break;
                }

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (!Mp || (*Mp == EOF_CHAR)) {
                diag_printf("Optional string [%s] not found\n", pstr);
                Mp = mp_save;
                Token_found_flag = 0;
                return 0;
        }

        Mp += strlen(pstr);
        diag_printf("Found optional string [%s]\n", pstr);
        Token_found_flag = 1;
        return 1;
}

int required_string_either(char *str1, char *str2)
{
        ignore_white_space();

        for (int count = 0; count < RS_MAX_TRIES; ++count) {
                if (strnicmp(str1, Mp, strlen(str1)) == 0) {
                        // Mp += strlen(str1);
                        diag_printf("Found required string [%s]\n", token_found = str1);
                        return 0;
                } else if (strnicmp(str2, Mp, strlen(str2)) == 0) {
                        // Mp += strlen(str2);
                        diag_printf("Found required string [%s]\n", token_found = str2);
                        return 1;
                }

                error_display(1, "Required token = [%s] or [%s], found [%.32s].\n", str1, str2, next_tokens());

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        nprintf(("Error", "Error: Unable to find either required token [%s] or [%s]\n", str1, str2));
        Warning(LOCATION, "Error: Unable to find either required token [%s] or [%s]\n", str1, str2);
        throw parse::ParseException("Required string not found");
        return -1;      // Dead code, but some compilers still complain about it
}

int required_string_one_of(int arg_count, ...)
{
        Assertion(arg_count > 0, "required_string_one_of() called with arg_count of %d; get a coder!\n", arg_count);
        int count = 0;
        int idx;
        char *expected;
        SCP_string message = "";
        va_list vl;

        ignore_white_space();

        while (count < RS_MAX_TRIES) {
                va_start(vl, arg_count);
                for (idx = 0; idx < arg_count; idx++) {
                        expected = va_arg(vl, char*);
                        if (strnicmp(expected, Mp, strlen(expected)) == 0) {
                                diag_printf("Found required string [%s]", token_found = expected);
                                va_end(vl);
                                return idx;
                        }
                }
                va_end(vl);

                if (!message.compare("")) {
                        va_start(vl, arg_count);
                        message = "Required token = ";
                        for (idx = 0; idx < arg_count; idx++) {
                                message += "[";
                                message += va_arg(vl, char*);
                                message += "]";
                                if (arg_count == 2 && idx == 0) {
                                        message += " or ";
                                } else if (idx == arg_count - 2) {
                                        message += ", or ";
                                } else if (idx < arg_count - 2) {
                                        message += ", ";
                                }
                        }
                        va_end(vl);
                }

                error_display(1, "%s, found [%.32s]\n", message.c_str(), next_tokens());
                advance_to_eoln(NULL);
                ignore_white_space();
                count++;
        }

        return -1;
}

int required_string_either_fred(char *str1, char *str2)
{
        ignore_white_space();

        while (*Mp != EOF_CHAR) {
                if (!strnicmp(str1, Mp, strlen(str1))) {
                        // Mp += strlen(str1);
                        diag_printf("Found required string [%s]\n", token_found = str1);
                        return fred_parse_flag = 0;

                } else if (!strnicmp(str2, Mp, strlen(str2))) {
                        // Mp += strlen(str2);
                        diag_printf("Found required string [%s]\n", token_found = str2);
                        return fred_parse_flag = 1;
                }

                advance_to_eoln(NULL);
                ignore_white_space();
        }

        if (*Mp == EOF_CHAR)
                diag_printf("Unable to find either required token [%s] or [%s]\n", str1, str2);

        return -1;
}

//      Copy characters from instr to outstr until eoln is found, or until max
//      characters have been copied (including terminator).
void copy_to_eoln(char *outstr, char *more_terminators, char *instr, int max)
{
        int     count = 0;
        char    ch;
        char    terminators[128];

        Assert((more_terminators == NULL) || (strlen(more_terminators) < 125));

        terminators[0] = EOLN;
        terminators[1] = EOF_CHAR;
        terminators[2] = 0;
        if (more_terminators != NULL)
                strcat_s(terminators, more_terminators);
        else
                terminators[2] = 0;

        while (((ch = *instr++) != 0) && (strchr(terminators, ch) == NULL)  && (count < max)) {
                *outstr++ = ch;
                count++;
        }

        if (count >= max)
                error_display(0, "Token too long: [%s].  Length = %i.  Max is %i.\n", next_tokens(), strlen(next_tokens()), max);

        *outstr = 0;
}

//      Ditto for SCP_string.
void copy_to_eoln(SCP_string &outstr, char *more_terminators, char *instr)
{
        char    ch;
        char    terminators[128];

        Assert((more_terminators == NULL) || (strlen(more_terminators) < 125));

        terminators[0] = EOLN;
        terminators[1] = EOF_CHAR;
        terminators[2] = 0;
        if (more_terminators != NULL)
                strcat_s(terminators, more_terminators);
        else
                terminators[2] = 0;

        outstr = "";
        while (((ch = *instr++) != 0) && (strchr(terminators, ch) == NULL)) {
                outstr.append(1, ch);
        }
}

//      Copy characters from instr to outstr until next white space is found, or until max
//      characters have been copied (including terminator).
void copy_to_next_white(char *outstr, char *instr, int max)
{
        int     count = 0;
        int     in_quotes = 0;
        char    ch;

        while (((ch = *instr++)>0) && (ch != EOLN) && (ch != EOF_CHAR) && (count < max)) {
                if ( ch == '\"' ) {
                        in_quotes = !in_quotes;
                        continue;
                }

                if ( !in_quotes && is_white_space(ch) ) // not in quotes, white space terminates string
                        break;

                if ( !in_quotes && is_parenthesis(ch) ) // not in quotes, parentheses are important for parsing so we don't want to copy them
                        break;

                *outstr++ = ch;
                count++;
        }

        if (count >= max)
                error_display(0, "Token too long: [%s].  Length = %i.  Max is %i.\n", next_tokens(), strlen(next_tokens()), max);

        *outstr = 0;
}

//      Ditto for SCP_string.
void copy_to_next_white(SCP_string &outstr, char *instr)
{
        int     in_quotes = 0;
        char    ch;

        outstr = "";
        while (((ch = *instr++)>0) && (ch != EOLN) && (ch != EOF_CHAR)) {
                if ( ch == '\"' ) {
                        in_quotes = !in_quotes;
                        continue;
                }

                if ( !in_quotes && is_white_space(ch) ) // not in quotes, white space terminates string
                        break;

                if ( !in_quotes && is_parenthesis(ch) ) // not in quotes, parentheses are important for parsing so we don't want to copy them
                        break;

                outstr.append(1, ch);
        }
}

//Returns a null-terminated character string allocated with vm_malloc() with the data
char* alloc_text_until(char* instr, char* endstr)
{
        Assert(instr && endstr);
        char *foundstr = stristr(instr, endstr);

        if(foundstr == NULL)
        {
        Error(LOCATION, "Missing [%s] in file", endstr);
        throw parse::ParseException("End string not found");
        }
        else
        {
                if ( (foundstr - instr) <= 0 ) {
                        Int3();  // since this really shouldn't ever happen
                        return NULL;
                }

                char* rstr = NULL;
                rstr = (char*) vm_malloc((foundstr - instr + 1)*sizeof(char));

                if(rstr != NULL) {
                        strncpy(rstr, instr, foundstr-instr);
                        rstr[foundstr-instr] = '\0';
                } else {
                        Error(LOCATION, "Could not allocate enough memory in alloc_text_until");
                }

                return rstr;
        }
}

//      Copy text until a certain string is matched.
//      For example, this is used to copy mission notes, scanning until $END NOTES:
// is found.
void copy_text_until(char *outstr, char *instr, char *endstr, int max_chars)
{
        char *foundstr;
        Assert(outstr && instr && endstr);

        foundstr = stristr(instr, endstr);

        if (foundstr == NULL) {
        nprintf(("Error", "Error.  Looking for [%s], but never found it.\n", endstr));
        throw parse::ParseException("End string not found");
        }

        if (foundstr - instr + strlen(endstr) < (uint) max_chars) {
                strncpy(outstr, instr, foundstr - instr);
                outstr[foundstr - instr] = 0;

        } else {
                nprintf(("Error", "Error.  Too much text (" SIZE_T_ARG " chars, %i allowed) before %s\n",
                        foundstr - instr - strlen(endstr), max_chars, endstr));

        throw parse::ParseException("Too much text found");
        }

        diag_printf("Here's the partial wad of text:\n%.30s\n", outstr);
}

//      Ditto for SCP_string.
void copy_text_until(SCP_string &outstr, char *instr, char *endstr)
{
        char *foundstr;
        Assert(instr && endstr);

        foundstr = stristr(instr, endstr);

        if (foundstr == NULL) {
        nprintf(("Error", "Error.  Looking for [%s], but never found it.\n", endstr));
        throw parse::ParseException("End string not found");
        }

        outstr.assign(instr, foundstr - instr);

        diag_printf("Here's the partial wad of text:\n%.30s\n", outstr.c_str());
}

// stuffs a string into a buffer.  Can get a string between " marks and stops
// when whitespace is encounted -- not to end of line
void stuff_string_white(char *outstr, int len)
{
        if(!len)
                len = NAME_LENGTH-1;

        ignore_white_space();
        copy_to_next_white(outstr, Mp, len);
        advance_to_next_white();
}

// ditto for SCP_string
void stuff_string_white(SCP_string &outstr)
{
        ignore_white_space();
        copy_to_next_white(outstr, Mp);
        advance_to_next_white();
}

// Goober5000
void stuff_string_until(char *outstr, char *endstr, int len)
{
        if(!len)
                len = NAME_LENGTH-1;

        ignore_gray_space();
        copy_text_until(outstr, Mp, endstr, len);
        Mp += strlen(outstr);
        drop_trailing_white_space(outstr);
}

// Goober5000
void stuff_string_until(SCP_string &outstr, char *endstr)
{
        ignore_gray_space();
        copy_text_until(outstr, Mp, endstr);
        Mp += outstr.length();
        drop_trailing_white_space(outstr);
}

//WMC
//Used for allocating large blocks, eg of Python code
//Returns a null-terminated string allocated with vm_malloc(),
//or NULL on failure
//Does depth checks for the start and end strings
//extra_chars indicates extra malloc space that should be allocated.
char* alloc_block(char* startstr, char* endstr, int extra_chars)
{
        Assert(startstr != NULL && endstr != NULL);
        Assert(stricmp(startstr, endstr));

        char* rval = NULL;
        uint elen = strlen(endstr);
        uint slen = strlen(startstr);
        uint flen = 0;

        //Skip the opening thing and any extra stuff
        required_string(startstr);
        ignore_white_space();

        //Allocate it
        char* pos = Mp;

        //Depth checking
        int level = 1;
        while(*pos != EOF_CHAR)
        {
                if(!strnicmp(pos, startstr, slen))
                {
                        level++;
                }
                else if(!strnicmp(pos, endstr, elen))
                {
                        level--;
                }

                if(level<=0)
                {
                        break;
                }

                pos++;
        }

        //Check that we left the file
        if(level > 0)
        {
        Error(LOCATION, "Unclosed pair of \"%s\" and \"%s\" on line %d in file", startstr, endstr, get_line_num());
        throw parse::ParseException("End string not found");
        }
        else
        {
                //Set final length for faster calcs
                flen = pos-Mp;

                //Allocate the memory
                //WMC - Don't forget the null character that's added later on.
                rval = (char*) vm_malloc((flen + extra_chars + 1)*sizeof(char));

                //Copy the text (if memory was allocated)
                if(rval != NULL) {
                        strncpy(rval, Mp, flen);
                        rval[flen] = '\0';
                } else {
                        return NULL;
                }
        }

        //Skip the copied stuff
        Mp += flen;
        required_string(endstr);
        return rval;
}

// Karajorma - Stuffs the provided char array with either the contents of a quoted string or the name of a string
// variable. Returns PARSING_FOUND_STRING if a string was found or PARSING_FOUND_VARIABLE if a variable was present.
int get_string_or_variable (char *str)
{
        int result = -1;

        ignore_white_space();

        // Variable
        if (*Mp == '@')
        {
                Mp++;
                stuff_string_white(str);
                int sexp_variable_index = get_index_sexp_variable_name(str);

                // We only want String variables
                Assertion (sexp_variable_index != -1, "Didn't find variable name \"%s\"", str);
                Assert (Sexp_variables[sexp_variable_index].type & SEXP_VARIABLE_STRING);

                result = PARSING_FOUND_VARIABLE;
        }
        // Quoted string
        else if (*Mp == '"')
        {
                get_string(str);
                result =  PARSING_FOUND_STRING;
        }
        else
        {
                get_string(str);
                Error(LOCATION, "Invalid entry \"%s\"  found in get_string_or_variable. Must be a quoted string or a string variable name.", str);
        }

        return result;
}

// ditto for SCP_string
int get_string_or_variable (SCP_string &str)
{
        int result = -1;

        ignore_white_space();

        // Variable
        if (*Mp == '@')
        {
                Mp++;
                stuff_string_white(str);
                int sexp_variable_index = get_index_sexp_variable_name(str);

                // We only want String variables
                Assertion (sexp_variable_index != -1, "Didn't find variable name \"%s\"", str.c_str());
                Assert (Sexp_variables[sexp_variable_index].type & SEXP_VARIABLE_STRING);

                result = PARSING_FOUND_VARIABLE;
        }
        // Quoted string
        else if (*Mp == '"')
        {
                get_string(str);
                result =  PARSING_FOUND_STRING;
        }
        else
        {
                get_string(str);
                Error(LOCATION, "Invalid entry \"%s\"  found in get_string_or_variable. Must be a quoted string or a string variable name.", str.c_str());
        }

        return result;
}

int get_string(char *str, int max)
{
        int     len;

        len = strcspn(Mp + 1, "\"");

        if (max >= 0 && len >= max)
                error_display(0, "String too long.  Length = %i.  Max is %i.\n", len, max);

        strncpy(str, Mp + 1, len);
        str[len] = 0;

        Mp += len + 2;
        return len;
}

void get_string(SCP_string &str)
{
        int len;

        len = strcspn(Mp + 1, "\"");
        str.assign(Mp + 1, len);

        Mp += len + 2;
}

//      Stuff a string into a string buffer.
//      Supports various FreeSpace primitive types.  If 'len' is supplied, it will override
// the default string length if using the F_NAME case.
void stuff_string(char *outstr, int type, int len, char *terminators)
{
        char read_str[PARSE_BUF_SIZE] = "";
        int read_len = PARSE_BUF_SIZE;
        int final_len = len - 1;
        int tag_id;

        // make sure we have enough room
        Assert( final_len > 0 );

        // make sure it's zero'd out
        memset( outstr, 0, len );

        switch (type) {
                case F_RAW:
                case F_LNAME:
                case F_NAME:
                case F_DATE:
                case F_FILESPEC:
                case F_PATHNAME:
                case F_MESSAGE:
                        ignore_gray_space();
                        copy_to_eoln(read_str, terminators, Mp, read_len);
                        drop_trailing_white_space(read_str);
                        advance_to_eoln(terminators);
                        break;

                case F_NOTES:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$End Notes:", read_len);
                        Mp += strlen(read_str);
                        required_string("$End Notes:");
                        break;

                // F_MULTITEXTOLD keeping for backwards compatability with old missions
                // can be deleted once all missions are using new briefing format

                case F_MULTITEXTOLD:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$End Briefing Text:", read_len);
                        Mp += strlen(read_str);
                        required_string("$End Briefing Text:");
                        break;

                case F_MULTITEXT:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$end_multi_text", read_len);
                        Mp += strlen(read_str);
                        drop_trailing_white_space(read_str);
                        required_string("$end_multi_text");
                        break;

                default:
                        Error(LOCATION, "Unhandled string type %d in stuff_string!", type);
        }
        
        if (type == F_FILESPEC) {
                // Make sure that the passed string looks like a good filename
                if (strlen(read_str) == 0) {
                        // Empty file name is not valid!
                        error_display(1, "A file name was expected but no name was supplied!\n");
                }
        }

        // now we want to do any final localization
        if(type != F_RAW && type != F_LNAME)
        {
                lcl_ext_localize(read_str, outstr, final_len, &tag_id);

                // if the hash localized text hash table is active and we have a valid external string - hash it
                if(fhash_active() && (tag_id > -2)){
                        fhash_add_str(outstr, tag_id);
                }
        }
        else
        {
                if ( strlen(read_str) > (uint)final_len )
                        error_display(0, "Token too long: [%s].  Length = %i.  Max is %i.\n", read_str, strlen(read_str), final_len);

                strncpy(outstr, read_str, final_len);
        }

        diag_printf("Stuffed string = [%.30s]\n", outstr);
}

//      Stuff a string into a string buffer.
//      Supports various FreeSpace primitive types.
void stuff_string(SCP_string &outstr, int type, char *terminators)
{
        SCP_string read_str;
        int tag_id;

        // make sure it's zero'd out
        outstr = "";

        switch (type) {
                case F_RAW:
                case F_LNAME:
                case F_NAME:
                case F_DATE:
                case F_FILESPEC:
                case F_PATHNAME:
                case F_MESSAGE:
                        ignore_gray_space();
                        copy_to_eoln(read_str, terminators, Mp);
                        drop_trailing_white_space(read_str);
                        advance_to_eoln(terminators);
                        break;

                case F_NOTES:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$End Notes:");
                        Mp += read_str.length();
                        required_string("$End Notes:");
                        break;

                // F_MULTITEXTOLD keeping for backwards compatability with old missions
                // can be deleted once all missions are using new briefing format

                case F_MULTITEXTOLD:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$End Briefing Text:");
                        Mp += read_str.length();
                        required_string("$End Briefing Text:");
                        break;

                case F_MULTITEXT:
                        ignore_white_space();
                        copy_text_until(read_str, Mp, "$end_multi_text");
                        Mp += read_str.length();
                        drop_trailing_white_space(read_str);
                        required_string("$end_multi_text");
                        break;

                default:
                        Error(LOCATION, "Unhandled string type %d in stuff_string!", type);
        }
        
        if (type == F_FILESPEC) {
                // Make sure that the passed string looks like a good filename
                if (read_str.empty()) {
                        // Empty file name is not valid!
                        error_display(1, "A file name was expected but no name was supplied!\n");
                }
        }

        // now we want to do any final localization
        if(type != F_RAW && type != F_LNAME)
        {
                lcl_ext_localize(read_str, outstr, &tag_id);

                // if the hash localized text hash table is active and we have a valid external string - hash it
                if(fhash_active() && (tag_id > -2)){
                        fhash_add_str(outstr.c_str(), tag_id);
                }
        }
        else
        {
                outstr = read_str;
        }

        diag_printf("Stuffed string = [%.30s]\n", outstr.c_str());
}

// stuff a string, but only until the end of a line. don't ignore leading whitespace. close analog of fgets()/cfgets()
void stuff_string_line(char *outstr, int len)
{
        char read_str[PARSE_BUF_SIZE] = "";
        int read_len = PARSE_BUF_SIZE;
        int final_len = len - 1;
        int tag_id;

        Assert( final_len > 0 );

        // read in a line
        copy_to_eoln(read_str, "\n", Mp, read_len);
        drop_trailing_white_space(read_str);
        advance_to_eoln("");
        Mp++;

        // now we want to do any final localization
        lcl_ext_localize(read_str, outstr, final_len, &tag_id);

        // if the hash localized text hash table is active and we have a valid external string - hash it
        if(fhash_active() && (tag_id > -2)){
                fhash_add_str(outstr, tag_id);
        }

        diag_printf("Stuffed string = [%.30s]\n", outstr);
}

// ditto for SCP_string
void stuff_string_line(SCP_string &outstr)
{
        SCP_string read_str;
        int tag_id;

        // read in a line
        copy_to_eoln(read_str, "\n", Mp);
        drop_trailing_white_space(read_str);
        advance_to_eoln("");
        Mp++;

        // now we want to do any final localization
        lcl_ext_localize(read_str, outstr, &tag_id);

        // if the hash localized text hash table is active and we have a valid external string - hash it
        if(fhash_active() && (tag_id > -2)){
                fhash_add_str(outstr.c_str(), tag_id);
        }

        diag_printf("Stuffed string = [%.30s]\n", outstr.c_str());
}

// Exactly the same as stuff string only Malloc's the buffer.
//      Supports various FreeSpace primitive types.  If 'len' is supplied, it will override
// the default string length if using the F_NAME case.
char *stuff_and_malloc_string(int type, char *terminators)
{
        SCP_string tmp_result;

        stuff_string(tmp_result, type, terminators);
        drop_white_space(tmp_result);

        if (tmp_result.empty())
                return NULL;

        return vm_strdup(tmp_result.c_str());
}

void stuff_malloc_string(char **dest, int type, char *terminators)
{
        Assert(dest != NULL); //wtf?

        char *new_val = stuff_and_malloc_string(type, terminators);

        if(new_val != NULL)
        {
                if((*dest) != NULL) {
                        vm_free(*dest);
                }

                (*dest) = new_val;
        }
}

// After reading a multitext string, you can call this function to convert any newlines into
// spaces, so it's a one paragraph string (I.e. as in MS-Word).
//
void compact_multitext_string(char *str)
{
        unsigned int i;
        unsigned int len = strlen(str);
        int num_cr = 0;

        for (i=0; i<len; i++)
        {
                // skip CR
                // convert LF to space
                // copy characters backwards if any CRs previously encountered
                if (str[i] == '\r')
                        num_cr++;
                else if (str[i] == '\n')
                        str[i-num_cr] = ' ';
                else if (num_cr > 0)
                        str[i-num_cr] = str[i];
        }

        if (num_cr > 0)
                str[len-num_cr] = 0;
}

// ditto for SCP_string
void compact_multitext_string(SCP_string &str)
{
        unsigned int i;
        unsigned int len = str.length();
        int num_cr = 0;

        for (i=0; i<len; i++)
        {
                // skip CR
                // convert LF to space
                // copy characters backwards if any CRs previously encountered
                if (str[i] == '\r')
                        num_cr++;
                else if (str[i] == '\n')
                        str[i-num_cr] = ' ';
                else if (num_cr > 0)
                        str[i-num_cr] = str[i];
        }

        if (num_cr > 0)
                str.resize(len-num_cr);
}

// Converts a character from Windows-1252 to CP437.
int maybe_convert_foreign_character(int ch)
{
        // time to do some special foreign character conversion
        switch (ch) {
                case -57:
                        ch = 128;
                        break;

                case -4:
                        ch = 129;
                        break;

                case -23:
                        ch = 130;
                        break;

                case -30:
                        ch = 131;
                        break;

                case -28:
                        ch = 132;
                        break;

                case -32:
                        ch = 133;
                        break;

                case -27:
                        ch = 134;
                        break;

                case -25:
                        ch = 135;
                        break;

                case -22:
                        ch = 136;
                        break;

                case -21:
                        ch = 137;
                        break;

                case -24:
                        ch = 138;
                        break;

                case -17:
                        ch = 139;
                        break;

                case -18:
                        ch = 140;
                        break;

                case -20:
                        ch = 141;
                        break;

                case -60:
                        ch = 142;
                        break;

                case -59:
                        ch = 143;
                        break;

                case -55:
                        ch = 144;
                        break;

                case -26:
                        ch = 145;
                        break;

                case -58:
                        ch = 146;
                        break;

                case -12:
                        ch = 147;
                        break;

                case -10:
                        ch = 148;
                        break;

                case -14:
                        ch = 149;
                        break;

                case -5:
                        ch = 150;
                        break;

                case -7:
                        ch = 151;
                        break;

                case -1:
                        ch = 152;
                        break;

                case -42:
                        ch = 153;
                        break;

                case -36:
                        ch = 154;
                        break;

                case -94:
                        ch = 155;
                        break;

                case -93:
                        ch = 156;
                        break;

                case -91:
                        ch = 157;
                        break;

                case -125:
                        ch = 159;
                        break;

                case -31:
                        ch = 160;
                        break;

                case -19:
                        ch = 161;
                        break;

                case -13:
                        ch = 162;
                        break;

                case -6:
                        ch = 163;
                        break;

                case -15:
                        ch = 164;
                        break;

                case -47:
                        ch = 165;
                        break;

                case -86:
                        ch = 166;
                        break;

                case -70:
                        ch = 167;
                        break;

                case -65:
                        ch = 168;
                        break;

                case -84:
                        ch = 170;
                        break;

                case -67:
                        ch = 171;
                        break;

                case -68:
                        ch = 172;
                        break;

                case -95:
                        ch = 173;
                        break;

                case -85:
                        ch = 174;
                        break;

                case -69:
                        ch = 175;
                        break;

                case -33:
                        ch = 225;
                        break;

                case -75:
                        ch = 230;
                        break;

                case -79:
                        ch = 241;
                        break;

                case -9:
                        ch = 246;
                        break;

                case -80:
                        ch = 248;
                        break;

                case -73:
                        ch = 250;
                        break;

                case -78:
                        ch = 253;
                        break;

                case -96:
                        ch = 255;
                        break;
        }

        return ch;
}

// Goober5000
void maybe_convert_foreign_characters(char *line)
{
        char *ch;
        if (Fred_running)
                return;

        if (Lcl_pl)
                return;

        for (ch = line; *ch != '\0'; ch++)
                        *ch = (char) maybe_convert_foreign_character(*ch);
}

// Goober5000
void maybe_convert_foreign_characters(SCP_string &line)
{
        if (Fred_running)
                return;

        if (Lcl_pl)
                return;

        for (SCP_string::iterator ii = line.begin(); ii != line.end(); ++ii)
                *ii = (char) maybe_convert_foreign_character(*ii);
}

// Goober5000
bool get_number_before_separator(int &number, int &number_chars, const char *text, char separator)
{
        char buf[8];
        const char *ch = text;
        int len = 0;

        while (true)
        {
                // didn't find separator
                if (*ch == '\0' || len == 8)
                        return false;

                // found separator
                if (*ch == separator)
                        break;

                // found nondigit
                if (!isdigit(*ch))
                        return false;

                // copying in progress
                buf[len] = *ch;
                len++;
                ch++;
        }

        // got an integer
        buf[len] = '\0';
        number = atoi(buf);
        number_chars = len;
        return true;
}

// Goober5000
bool get_number_before_separator(int &number, int &number_chars, const SCP_string &text, SCP_string::iterator text_pos, char separator)
{
        char buf[8];
        SCP_string::iterator ch = text_pos;
        int len = 0;

        while (true)
        {
                // didn't find separator
                if (ch == text.end() || len == 8)
                        return false;

                // found separator
                if (*ch == separator)
                        break;

                // found nondigit
                if (!isdigit(*ch))
                        return false;

                // copying in progress
                buf[len] = *ch;
                len++;
                ++ch;
        }

        // got an integer
        buf[len] = '\0';
        number = atoi(buf);
        number_chars = len;
        return true;
}

bool matches_version_specific_tag(const char *line_start, bool &compatible_version, int &tag_len)
{
        // special version-specific comment
        // formatted like e.g. ;;FSO 3.7.0;;
        // Should now support anything from ;;FSO 3;; to ;;FSO 3.7.3.20151106;; -MageKing17
        if (strnicmp(line_start, ";;FSO ", 6))
                return false;

        int major, minor, build, revis;
        int s_num = scan_fso_version_string(line_start, &major, &minor, &build, &revis);

        if (s_num == 0)
                return false;

        // hack for releases
        if (s_num == 4 && FS_VERSION_REVIS < 1000) {
                s_num = 3;
        }

        const char *ch = line_start + 6;
        while ((*ch) != ';') {
                Assertion((*ch) != '\0', "String that was already guaranteed to end with semicolons did not end with semicolons; it's possible we have fallen into an alternate universe. Failing string: [%s]\n", line_start);
                ch++;
        }
        ch++;
        Assertion((*ch) == ';', "String that was guaranteed to have double semicolons did not; it's possible we have fallen into an alternate universe. Failing string: [%s]\n", line_start);
        ch++;

        tag_len = ch - line_start;
        compatible_version = true;

        // check whether major, minor, and build line up with this version
        if (major > FS_VERSION_MAJOR)
        {
                compatible_version = false;
        }
        else if (major == FS_VERSION_MAJOR && s_num > 1)
        {
                if (minor > FS_VERSION_MINOR)
                {
                        compatible_version = false;
                }
                else if (minor == FS_VERSION_MINOR && s_num > 2)
                {
                        if (build > FS_VERSION_BUILD)
                        {
                                compatible_version = false;
                        }
                        else if (build == FS_VERSION_BUILD && s_num > 3)
                        {
                                if (revis > FS_VERSION_REVIS)
                                {
                                        compatible_version = false;
                                }
                        }
                }
        }

        // true for tag match
        return true;
}

// Strip comments from a line of input.
// Goober5000 - rewritten for the second time
void strip_comments(char *line, bool &in_quote, bool &in_multiline_comment_a, bool &in_multiline_comment_b)
{
        char *writep = line;
        char *readp = line;

        // copy all characters from read to write, unless they're commented
        while (*readp != '\r' && *readp != '\n' && *readp != '\0')
        {
                // only check for comments if not quoting
                if (!in_quote)
                {
                        bool compatible_version;
                        int tag_len;

                        // see what sort of comment characters we recognize
                        if (!strncmp(readp, "/*", 2))
                        {
                                // comment styles are mutually exclusive
                                if (!in_multiline_comment_b)
                                        in_multiline_comment_a = true;
                        }
                        else if (!strncmp(readp, "!*", 2))
                        {
                                // comment styles are mutually exclusive
                                if (!in_multiline_comment_a)
                                        in_multiline_comment_b = true;
                        }
                        else if (!strncmp(readp, "*/", 2))
                        {
                                if (in_multiline_comment_a)
                                {
                                        in_multiline_comment_a = false;
                                        readp += 2;
                                        continue;
                                }
                        }
                        else if (!strncmp(readp, "*!", 2))
                        {
                                if (in_multiline_comment_b)
                                {
                                        in_multiline_comment_b = false;
                                        readp += 2;
                                        continue;
                                }
                        }
                        // special version-specific comment
                        // formatted like e.g. ;;FSO 3.7.0;;
                        else if (matches_version_specific_tag(readp, compatible_version, tag_len))
                        {
                                // comment passes, so advance pass the tag and keep reading
                                if (compatible_version)
                                {
                                        readp += tag_len;
                                        continue;
                                }
                                // comment does not pass, so ignore the line
                                else
                                {
                                        break;
                                }
                        }
                        // standard comment
                        else if (*readp == ';')
                        {
                                break;
                        }
                }

                // maybe toggle quoting
                if (*readp == '\"')
                        in_quote = !in_quote;

                // if not inside a comment, copy the characters
                if (!in_multiline_comment_a && !in_multiline_comment_b)
                {
                        if (writep != readp)
                                *writep = *readp;

                        writep++;
                }

                // read the next character
                readp++;
        }

        // if we moved any characters, or if we haven't reached the end of the string, then mark end-of-line and terminate string
        if (writep != readp || *readp != '\0')
        {
                writep[0] = EOLN;
                writep[1] = '\0';
        }
}

int parse_get_line(char *lineout, int max_line_len, char *start, int max_size, char *cur)
{
        char * t = lineout;
        int i, num_chars_read=0;
        char c;


        for ( i = 0; i < max_line_len-1; i++ ) {
                do {
                        if ( (cur - start) >= max_size ) {
                                *lineout = 0;
                                if ( lineout > t ) {
                                        return num_chars_read;
                                } else {
                                        return 0;
                                }
                        }
                        c = *cur++;
                        num_chars_read++;
                } while ( c == 13 );

                *lineout++ = c;
                if ( c=='\n' ) break;
        }

        *lineout++ = 0;
        return  num_chars_read;
}

//      Read mission text, stripping comments.
//      When a comment is found, it is removed.  If an entire line
//      consisted of a comment, a blank line is left in the input file.
// Goober5000 - added ability to read somewhere other than Mission_text
void read_file_text(const char *filename, int mode, char *processed_text, char *raw_text)
{
        // copy the filename
    if (!filename)
        throw parse::ParseException("Invalid filename");

        strcpy_s(Current_filename_sub, filename);

        // if we are paused then processed_text and raw_text must not be NULL!!
        if ( Parsing_paused && ((processed_text == NULL) || (raw_text == NULL)) ) {
                Error(LOCATION, "ERROR: Neither processed_text nor raw_text may be NULL when parsing is paused!!\n");
        }

        // read the raw text
        read_raw_file_text(filename, mode, raw_text);

        if (processed_text == NULL)
                processed_text = Mission_text;

        if (raw_text == NULL)
                raw_text = Mission_text_raw;

        // process it (strip comments)
        process_raw_file_text(processed_text, raw_text);
}

// Goober5000
void read_file_text_from_array(const char *array, char *processed_text, char *raw_text)
{
        // we have no filename, so copy a substitute
        strcpy_s(Current_filename_sub, "internal default file");

        // if we are paused then processed_text and raw_text must not be NULL!!
        if ( Parsing_paused && ((processed_text == NULL) || (raw_text == NULL)) ) {
                Error(LOCATION, "ERROR: Neither \"processed_text\" nor \"raw_text\" may be NULL when parsing is paused!!\n");
        }

        // make sure to do this before anything else
        allocate_mission_text( strlen(array) + 1 );

        // if we have no raw buffer, set it as the default raw text area
        if (raw_text == NULL)
                raw_text = Mission_text_raw;

        // copy text in the array (but only if the raw text and the array are not the same)
        if (raw_text != array)
                strcpy(raw_text, array);

        if (processed_text == NULL)
                processed_text = Mission_text;

        // process the text
        process_raw_file_text(processed_text, raw_text);
}

// Goober5000
int is_unicode(char *text)
{
        if (!strncmp(text, "\xEF\xBB\xBF", 3))          // UTF-8
                return 1;

        if (!strncmp(text, "\xFE\xFF", 2))                      // UTF-16 big-endian
                return 1;

        if (!strncmp(text, "\xFF\xFE", 2))                      // UTF-16 little-endian
                return 1;

        if (!strncmp(text, "\x00\x00\xFE\xFF", 4))      // UTF-32 big-endian
                return 1;

        if (!strncmp(text, "\xFF\xFE\x00\x00", 4))      // UTF-32 little-endian
                return 1;

        return 0;
}

void stop_parse()
{
        Assert( !Parsing_paused );

        if (Mission_text != NULL) {
                vm_free(Mission_text);
                Mission_text = NULL;
        }

        if (Mission_text_raw != NULL) {
                vm_free(Mission_text_raw);
                Mission_text_raw = NULL;
        }

        Mission_text_size = 0;
}

void allocate_mission_text(int size)
{
        Assert( size > 0 );

        if (size <= Mission_text_size)
                return;


        static ubyte parse_atexit = 0;

        if (!parse_atexit) {
                atexit(stop_parse);
                parse_atexit = 1;
        }

        if (Mission_text != NULL) {
                vm_free(Mission_text);
                Mission_text = NULL;
        }

        if (Mission_text_raw != NULL) {
                vm_free(Mission_text_raw);
                Mission_text_raw = NULL;
        }

        Mission_text = (char *) vm_malloc_q(sizeof(char) * size);
        Mission_text_raw = (char *) vm_malloc_q(sizeof(char) * size);

        if ( (Mission_text == NULL) || (Mission_text_raw == NULL) ) {
                Error(LOCATION, "Unable to allocate enough memory for Mission_text!  Aborting...\n");
        }

        memset( Mission_text, 0, sizeof(char) * size );
        memset( Mission_text_raw, 0, sizeof(char) * size);

        Mission_text_size = size;
}

// Goober5000
void read_raw_file_text(const char *filename, int mode, char *raw_text)
{
        CFILE   *mf;
        int     file_is_encrypted;
        int file_is_unicode;

        Assert(filename);

        mf = cfopen(filename, "rb", CFILE_NORMAL, mode);
        if (mf == NULL)
        {
        nprintf(("Error", "Wokka!  Error opening file (%s)!\n", filename));
        throw parse::ParseException("Failed to open file");
        }

        // read the entire file in
        int file_len = cfilelength(mf);

        if(!file_len) {
        nprintf(("Error", "Oh noes!!  File is empty! (%s)!\n", filename));
        throw parse::ParseException("Failed to open file");
        }

        // allocate, or reallocate, memory for Mission_text and Mission_text_raw based on size we need now
        allocate_mission_text( file_len + 1 );

        // NOTE: this always has to be done *after* the allocate_mission_text() call!!
        if (raw_text == NULL)
                raw_text = Mission_text_raw;

        // read first 10 bytes to determine if file is encrypted
        cfread(raw_text, MIN(file_len, 10), 1, mf);
        file_is_encrypted = is_encrypted(raw_text);
        cfseek(mf, 0, CF_SEEK_SET);

        // Goober5000 - also determine if file is Unicode
        file_is_unicode = is_unicode(raw_text);
        if ( file_is_unicode )
        {
                //This is probably fatal, so let's abort right here and now.
                Error(LOCATION, "%s is in Unicode/UTF format and cannot be read by FreeSpace Open. Please convert it to ASCII/ANSI\n", filename);
        }

        if ( file_is_encrypted )
        {
                int     unscrambled_len;
                char    *scrambled_text;
                scrambled_text = (char*)vm_malloc(file_len+1);
                Assert(scrambled_text);
                cfread(scrambled_text, file_len, 1, mf);
                // unscramble text
                unencrypt(scrambled_text, file_len, raw_text, &unscrambled_len);
                file_len = unscrambled_len;
                vm_free(scrambled_text);
        }
        else
        {
                cfread(raw_text, file_len, 1, mf);
        }

        //WMC - Slap a NULL character on here for the odd error where we forgot a #End
        raw_text[file_len] = '\0';

        cfclose(mf);
}

// Goober5000
void process_raw_file_text(char *processed_text, char *raw_text)
{
        char    *mp;
        char    *mp_raw;
        char outbuf[PARSE_BUF_SIZE], *str;
        bool in_quote = false;
        bool in_multiline_comment_a = false;
        bool in_multiline_comment_b = false;
        int raw_text_len = strlen(raw_text);

        if (processed_text == NULL)
                processed_text = Mission_text;

        if (raw_text == NULL)
                raw_text = Mission_text_raw;

        Assert( processed_text != NULL );
        Assert( raw_text != NULL );

        mp = processed_text;
        mp_raw = raw_text;

        // strip comments from raw text, reading into file_text
        int num_chars_read = 0;
        while ( (num_chars_read = parse_get_line(outbuf, PARSE_BUF_SIZE, raw_text, raw_text_len, mp_raw)) != 0 ) {
                mp_raw += num_chars_read;

                // stupid hacks to make retail data work with fixed parser, per Mantis #3072
                if (!strcmp(outbuf, "1402, \"Sie haben IPX-Protokoll als Protokoll ausgew\xE4hlt, aber dieses Protokoll ist auf Ihrer Maschine nicht installiert.\".\"\n")) {
                        outbuf[121] = ' ';
                        outbuf[122] = ' ';
                } else if (!strcmp(outbuf, "1117, \"\\r\\n\"Aucun web browser trouva. Del\xE0 isn't on emm\xE9nagea ou if \\r\\non est emm\xE9nagea, ca isn't set pour soient la default browser.\\r\\n\\r\\n\"\n")) {
                        char *ch = &outbuf[11];
                        do {
                                *ch = *(ch+1);
                                ++ch;
                        } while (*ch);
                } else if (!strcmp(outbuf, "1337, \"(fr)Loading\"\n")) {
                        outbuf[3] = '6';
                } else if (!strcmp(outbuf, "3966, \"Es sieht so aus, als habe Staffel Kappa Zugriff auf die GTVA-Zugangscodes f\xFCr das System gehabt. Das ist ein ernstes Sicherheitsleck. Ihre IFF-Kennung erschien als \"verb\xFCndet\", so da\xDF sie sich dem Konvoi ungehindert n\xE4hern konnten. Zum Gl\xFC\x63k flogen Sie und  Alpha 2 Geleitschutz und lie\xDF\x65n den Schwindel auffliegen, bevor Kappa ihren Befehl ausf\xFChren konnte.\"\n")) {
                        outbuf[171] = '\'';
                        outbuf[181] = '\'';
                }

                strip_comments(outbuf, in_quote, in_multiline_comment_a, in_multiline_comment_b);

                maybe_convert_foreign_characters(outbuf);

                str = outbuf;
                while (*str) {
                        if (*str == (Lcl_pl ? -33 : -31)) {
                                *mp++ = 's';
                                *mp++ = 's';
                                str++;

                        } else {
                                *mp++ = *str++;
                        }
                }

//              strcpy_s(mp, outbuf);
//              mp += strlen(outbuf);
        }

        *mp = *mp_raw = EOF_CHAR;
/*
        while (cfgets(outbuf, PARSE_BUF_SIZE, mf) != NULL) {
                if (strlen(outbuf) >= PARSE_BUF_SIZE-1)
                        error_display(0, "Input string too long.  Max is %i characters.\n%.256s\n", PARSE_BUF_SIZE, outbuf);

                //      If you hit this assert, it is probably telling you the obvious.  The file
                //      you are trying to read is truly too large.  Look at *filename to see the file name.
                Assert(mp_raw - file_text_raw + strlen(outbuf) < MISSION_TEXT_SIZE);
                strcpy_s(mp_raw, outbuf);
                mp_raw += strlen(outbuf);

                in_comment = strip_comments(outbuf, in_comment);
                strcpy_s(mp, outbuf);
                mp += strlen(outbuf);
        }

        *mp = *mp_raw = EOF_CHAR;
*/

}

void debug_show_mission_text()
{
        char    *mp = Mission_text;
        char    ch;

        while ((ch = *mp++) != EOF_CHAR)
                printf("%c", ch);
}

float atof2()
{
        char    ch;

        my_errno = 0;
        ignore_white_space();

        ch = *Mp;

        if ((ch != '.') && (ch != '-') && (ch != '+') && ((ch < '0') || (ch > '9'))) {
                error_display(1, "Expecting float, found [%.32s].\n", next_tokens());
                my_errno = 1;
                return 0.0f;
        } else
                return (float)atof(Mp);

}

int atoi2()
{
        char    ch;

        my_errno = 0;

        ignore_white_space();

        ch = *Mp;

        if ((ch != '-') && (ch != '+') && ((ch < '0') || (ch > '9'))) {
                error_display(1, "Expecting int, found [%.32s].\n", next_tokens());
                my_errno = 1;
                return 0;
        } else
                return atoi(Mp);

}

//      Stuff a floating point value pointed at by Mp.
//      Advances past float characters.
void stuff_float(float *f)
{
        *f = atof2();

        if (my_errno)
                skip_token();
        else
                Mp += strspn(Mp, "+-0123456789.");

        if (*Mp ==',')
                Mp++;

        diag_printf("Stuffed float: %f\n", *f);
}

int stuff_float_optional(float *f, bool raw)
{
        int skip_len;
        bool comma = false;

        if (!raw)
                ignore_white_space();

        skip_len = strspn(Mp, "+-0123456789.");
        if(*(Mp+skip_len) == ',') {
                comma = true;
        }

        if(skip_len == 0)
        {
                if(comma) {
                        Mp++;
                        return 1;
                } else {
                        return 0;
                }
        }

        stuff_float(f);
        return 2;
}

//      Stuff an integer value pointed at by Mp.
//      Advances past integer characters.
void stuff_int(int *i)
{
        *i = atoi2();

        if (my_errno)
                skip_token();
        else
                Mp += strspn(Mp, "+-0123456789");

        if (*Mp ==',')
                Mp++;

        diag_printf("Stuffed int: %i\n", *i);
}

int stuff_int_optional(int *i, bool raw)
{
        int skip_len;
        bool comma = false;

        if (!raw)
                ignore_white_space();

        skip_len = strspn(Mp, "+-0123456789");
        if(*(Mp+skip_len) == ',') {
                comma = true;
        }

        if(skip_len == 0)
        {
                if(comma) {
                        Mp++;
                        return 1;
                } else {
                        return 0;
                }
        }

        stuff_int(i);
        return 2;
}

int stuff_int_or_variable (int &i, bool positive_value = false);
int stuff_int_or_variable (int *ilp, int count, bool positive_value = false);

// Stuffs an int value or the value of a number variable. Returns the index of the variable or NOT_SET_BY_SEXP_VARIABLE.
int stuff_int_or_variable (int &i, bool positive_value)
{
        int index = NOT_SET_BY_SEXP_VARIABLE;

        if (*Mp == '@')
        {
                Mp++;
                int value = -1;
                char str[128];
                stuff_string(str, F_NAME, sizeof(str));

                index = get_index_sexp_variable_name(str);

                if (index > -1 && index < MAX_SEXP_VARIABLES)
                {
                        if (Sexp_variables[index].type & SEXP_VARIABLE_NUMBER)
                        {
                                value = atoi(Sexp_variables[index].text);
                        }
                        else
                        {
                                Error(LOCATION, "Invalid variable type \"%s\" found in mission. Variable must be a number variable!", str);
                        }
                }
                else
                {

                        Error(LOCATION, "Invalid variable name \"%s\" found.", str);
                }

                // zero negative values if requested
                if (positive_value && value < 0)
                {
                        value = 0;
                }


                // Record the value of the index for FreeSpace
                i = value;
        }
        else
        {
                stuff_int(&i);
        }
        return index;
}

// Stuff an integer value pointed at by Mp.If a variable is found instead stuff the value of that variable and record the
// index of the variable in the following slot.
int stuff_int_or_variable (int *ilp, int count, bool positive_value)
{
        if (*Mp == '@')
        {
                Mp++;
                int value = -1;
                char str[128];
                stuff_string(str, F_NAME, sizeof(str));

                int index = get_index_sexp_variable_name(str);

                if (index > -1 && index < MAX_SEXP_VARIABLES)
                {
                        if (Sexp_variables[index].type & SEXP_VARIABLE_NUMBER)
                        {
                                value = atoi(Sexp_variables[index].text);
                        }
                        else
                        {
                                Error(LOCATION, "Invalid variable type \"%s\" found in mission. Variable must be a number variable!", str);
                        }
                }
                else
                {

                        Error(LOCATION, "Invalid variable name \"%s\" found.", str);
                }

                // zero negative values if requested
                if (positive_value && value < 0)
                {
                        value = 0;
                }


                // Record the value of the index for FreeSpace
                ilp[count++] = value;
                // Record the index itself because we may need it later.
                ilp[count++] = index;
        }
        else
        {
                stuff_int(&ilp[count++]);
                // Since we have a numerical value we don't have a SEXP variable index to add for next slot.
                ilp[count++] = NOT_SET_BY_SEXP_VARIABLE;
        }
        return count;
}


//Stuffs boolean value.
//Passes things off to stuff_boolean(bool)
void stuff_boolean(int *i, bool a_to_eol)
{
        bool tempb;
        stuff_boolean(&tempb, a_to_eol);
        if(tempb)
                *i = 1;
        else
                *i = 0;
}

void stuff_boolean_flag(int *i, int flag, bool a_to_eol)
{
        bool temp;
        stuff_boolean(&temp, a_to_eol);
        if(temp)
                *i |= flag;
        else
                *i &= ~(flag);
}

// Stuffs a boolean value pointed at by Mp.
// YES/NO (supporting 1/0 now as well)
// Now supports localization :) -WMC

void stuff_boolean(bool *b, bool a_to_eol)
{
        char token[32];
        stuff_string_white(token, sizeof(token)/sizeof(char));
        if(a_to_eol)
                advance_to_eoln(NULL);

        if( isdigit(token[0]))
        {
                if(token[0] != '0')
                        *b = true;
                else
                        *b = false;
        }
        else
        {
                if(!stricmp(token, "yes")
                        || !stricmp(token, "true")
                        || !stricmp(token, "ja")                //German
                        || !stricmp(token, "Oui")               //French
                        || !stricmp(token, "si")                //Spanish
                        || !stricmp(token, "ita vero")  //Latin
                        || !stricmp(token, "HIja'") || !stricmp(token, "HISlaH"))       //Klingon
                {
                        *b = true;
                }
                else if(!stricmp(token, "no")
                        || !stricmp(token, "false")
                        || !stricmp(token, "nein")              //German
                        || !stricmp(token, "Non")               //French
                        //I don't know spanish for "no"
                        //But according to altavista, spanish for "No" is "no"
                        //Go figure.
                        || !stricmp(token, "minime")    //Latin
                        || !stricmp(token, "ghobe'"))   //Klingon
                {
                        *b = false;
                }
                else
                {
                        *b = false;
                        Warning(LOCATION, "Boolean '%s' type unknown; assuming 'no/false'",token);
                }
        }

        diag_printf("Stuffed bool: %s\n", (b) ? NOX("true") : NOX("false"));
}

int stuff_bool_list(bool *blp, int max_bools)
{
        int count = 0;
        bool trash_buf = false;

        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading boolean list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;

        ignore_white_space();

        while(*Mp != ')')
        {
                if(count < max_bools)
                {
                        stuff_boolean(&blp[count++], false);
                        ignore_white_space();

                        //Since Bobb has set a precedent, allow commas for bool lists -WMC
                        if(*Mp == ',')
                        {
                                Mp++;
                                ignore_white_space();
                        }
                }
                else
                {
                        trash_buf = true;
                        break;
                }
        }

        if(trash_buf)
        {
                error_display(0, "Boolean list has more than allowed arguments; max is %d. Arguments over max will be ignored.", max_bools);
                while(*Mp != ')')
                {
                        stuff_boolean(&trash_buf, false);
                        ignore_white_space();
                }
        }

        Mp++;

        return count;
}


//      Stuff an integer value pointed at by Mp.
//      Advances past integer characters.
void stuff_ubyte(ubyte *i)
{
        int     temp;

        temp = atoi2();

        *i = (ubyte)temp;

        if (my_errno)
                skip_token();
        else
                Mp += strspn(Mp, "+-0123456789");

        if (*Mp == ',')
                Mp++;

        diag_printf("Stuffed byte: %i\n", *i);
}

int parse_string_flag_list(int *dest, flag_def_list defs[], int defs_size)
{
        Assert(dest!=NULL);     //wtf?

        char (*slp)[NAME_LENGTH] = (char(*)[32])new char[defs_size*NAME_LENGTH];
        int num_strings = stuff_string_list(slp, defs_size);
        int i, j;

        for(i = 0; i < num_strings; i++)
        {
                for(j = 0; j < defs_size; j++)
                {
                        if(!stricmp(slp[i], defs[j].name)) {
                                (*dest) |= defs[j].def;
                        }
                }
        }

        delete[] slp;   //>_>
        //nobody saw that right

        return num_strings;
}

int stuff_string_list(SCP_vector<SCP_string>& slp)
{
        //_asm int 3;
        slp.clear();

        ignore_white_space();

        if ( *Mp != '(' ) {
        error_display(1, "Reading string list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;

        ignore_white_space();

        SCP_string buf;

        while (*Mp != ')') {
                if(*Mp != '\"') {
                        error_display(0, "Missing quotation marks in string list.");
                }
                //Assert ( *Mp == '\"' );                                       // should always be enclosed in quotes

                buf = "";
                get_string( buf );
                slp.push_back( buf );
                ignore_white_space();
        }

        Mp++;

        return slp.size();
}

// Stuffs a list of strings
int stuff_string_list(char slp[][NAME_LENGTH], int max_strings)
{
        int count = 0;
        ignore_white_space();

        if ( *Mp != '(' ) {
        error_display(1, "Reading string list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;

        ignore_white_space();

        while (*Mp != ')') {
                Assert ( count < max_strings );
                if(*Mp != '\"') {
                        error_display(0, "Missing quotation marks in string list.");
                }
                //Assert ( *Mp == '\"' );                                       // should always be enclosed in quotes

                if (count < max_strings) {
                        get_string( slp[count++] );
                } else {
                        char trash[NAME_LENGTH];
                        get_string( trash );
                }
                ignore_white_space();
        }

        Mp++;

        return count;
}

const char* get_lookup_type_name(int lookup_type)
{
        switch (lookup_type) {
                case SHIP_TYPE:
                        return "Ships";
                case SHIP_INFO_TYPE:
                        return "Ship Classes";
                case WEAPON_POOL_TYPE:
                        return "Weapon Pool";
                case WEAPON_LIST_TYPE:
                        return "Weapon Types";
                case RAW_INTEGER_TYPE:
                        return "Untyped integer list";
        }

        return "Unknown lookup type, tell a coder!";
}

//      Stuffs an integer list.
//      This is of the form ( i* )
//        where i is an integer.
// For example, (1) () (1 2 3) ( 1 ) are legal integer lists.
int stuff_int_list(int *ilp, int max_ints, int lookup_type)
{
        int     count = 0, ok_flag = 1, dummy;
        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading integer list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;
        ignore_white_space();

        while (*Mp != ')') {
                Assertion(count < max_ints, "Too many entries in integer list. Expected %d, found %d.\nList type was %s", max_ints, count+1, get_lookup_type_name(lookup_type));
                if (*Mp == '"') {
                        int num = 0;
                        char str[128];

                        get_string(str);
                        switch (lookup_type) {
                                case SHIP_TYPE:
                                        num = ship_name_lookup(str);    // returns index of Ship[] entry with name
                                        break;

                                case SHIP_INFO_TYPE:
                                        ok_flag = 1;
                                        num = ship_info_lookup(str);    // returns index of Ship_info[] entry with name
                                        if (num < 0)
                                                ok_flag = 0;
                                        break;

                                case WEAPON_POOL_TYPE:
                                        ok_flag = 1;
                                        num = weapon_info_lookup(str);
                                        if (num < 0)
                                                ok_flag = 0;
                                        break;

                                case WEAPON_LIST_TYPE:
                                        num = weapon_info_lookup(str);
                                        if (num < 0)
                                                num = -2;
                                        break;

                                case RAW_INTEGER_TYPE:
                                        num = atoi(str);
                                        break;

                                default:
                                        Error(LOCATION,"Unknown lookup_type in stuff_int_list");
                                        break;
                        }

                        if (ok_flag) {
                                if (num == -1) {
                                        Error(LOCATION, "Unable to find string \"%s\" in stuff_int_list\n\nMany possible sources for this error.  Get a programmer!\n", str);
                                } else if (num == -2) {
                                        if (str[0] != '\0') {
                                                Warning(LOCATION, "Unable to find WEAPON_LIST_TYPE string \"%s\" in stuff_int_list\n\nMany possible sources for this error.  Get a programmer!\n", str);
                                        }
                                }

                                if (num < 0)  // other negatives used to bypass the above error trap, but should be -1
                                        num = -1;

                                if (count < max_ints) {
                                        ilp[count++] = num;
                                }
                        }

                } else {
                        if (ok_flag && (count < max_ints))
                                stuff_int(&ilp[count++]);
                        else
                                stuff_int(&dummy);
                }

                ignore_white_space();
        }

        Mp++;

        return count;
}

// helper for the next function. Removes a broken entry from ship or weapon lists and advances to the next one
void clean_loadout_list_entry()
{
        int dummy;

        // clean out the broken entry
        ignore_white_space();
        stuff_int_or_variable(dummy);
        ignore_white_space();
}

// Karajorma - Stuffs an int list by parsing a list of ship or weapon choices.
// Unlike stuff_int_list it can deal with variables and it also has better error reporting.
int stuff_loadout_list (int *ilp, int max_ints, int lookup_type)
{
        int count = 0;
        int index, sexp_variable_index, variable_found;
        char str[128];

        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading loadout list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;
        ignore_white_space();

        while (*Mp != ')') {
                if (count >= max_ints) {
                        Error(LOCATION, "Loadout contains too many entries.\n");
                }

                index = -1;
                sexp_variable_index = NOT_SET_BY_SEXP_VARIABLE;
                variable_found = get_string_or_variable (str);

                // if we've got a variable get the variable index and copy it's value into str so that regardless of whether we found
                // a variable or not it now holds the name of the ship or weapon we're interested in.
                if (variable_found) {
                        Assert (lookup_type != CAMPAIGN_LOADOUT_SHIP_LIST );
                        sexp_variable_index = get_index_sexp_variable_name(str);

                        if(sexp_variable_index<0) {
                                Error(LOCATION, "Invalid SEXP variable name \"%s\" found in stuff_loadout_list.", str);
                        }

                        strcpy_s (str, Sexp_variables[sexp_variable_index].text);
                }

                switch (lookup_type) {
                        case MISSION_LOADOUT_SHIP_LIST:
                        case CAMPAIGN_LOADOUT_SHIP_LIST:
                                index = ship_info_lookup(str);
                                break;

                        case MISSION_LOADOUT_WEAPON_LIST:
                        case CAMPAIGN_LOADOUT_WEAPON_LIST:
                                index = weapon_info_lookup(str);
                                break;

                        default:
                                Int3();
                }

                // Complain if this isn't a valid ship or weapon and we are loading a mission. Campaign files can be loading containing
                // no ships from the current tables (when swapping mods) so don't report that as an error.
                if (index < 0 && (lookup_type == MISSION_LOADOUT_SHIP_LIST || lookup_type == MISSION_LOADOUT_WEAPON_LIST)) {
                        // print a warning in debug mode
                        Warning(LOCATION, "Invalid type \"%s\" found in loadout of mission file...skipping", str);
                        // increment counter for release FRED builds.
                        Num_unknown_loadout_classes++;

                        clean_loadout_list_entry();
                        continue;
                }

                // similarly, complain if this is a valid ship or weapon class that the player can't use
                if ((lookup_type == MISSION_LOADOUT_SHIP_LIST) && (!(Ship_info[index].flags & SIF_PLAYER_SHIP)) ) {
                        clean_loadout_list_entry();
                        Warning(LOCATION, "Ship type \"%s\" found in loadout of mission file. This class is not marked as a player ship...skipping", str);
                        continue;
                }
                else if ((lookup_type == MISSION_LOADOUT_WEAPON_LIST) && (!(Weapon_info[index].wi_flags & WIF_PLAYER_ALLOWED)) ) {
                        clean_loadout_list_entry();
                        nprintf(("Warning",  "Warning: Weapon type %s found in loadout of mission file. This class is not marked as a player allowed weapon...skipping\n", str));
                        if ( !Is_standalone )
                                Warning(LOCATION, "Weapon type \"%s\" found in loadout of mission file. This class is not marked as a player allowed weapon...skipping", str);
                        continue;
                }

                // we've found a real item. Add its index to the list.
                if (count < max_ints) {
                        ilp[count++] = index;
                }

                ignore_white_space();

                // Campaign lists need go no further
                if (lookup_type == CAMPAIGN_LOADOUT_SHIP_LIST || lookup_type == CAMPAIGN_LOADOUT_WEAPON_LIST) {
                        continue;
                }

                // record the index of the variable that gave us this item if any
                if (count < max_ints) {
                        ilp[count++] = sexp_variable_index;
                }

                // Now read in the number of this type available. The number must be positive
                count = stuff_int_or_variable(ilp, count, true);

                ignore_white_space();
        }

        Mp++;
        return count;
}

//Stuffs an integer list like stuff_int_list.
int stuff_float_list(float* flp, int max_floats)
{
        int count = 0;
        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading float list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;
        ignore_white_space();
        while(*Mp != ')')
        {
                Assert(count < max_floats);
                if (count < max_floats) {
                        stuff_float(&flp[count++]);
                } else {
                        float dummy;
                        stuff_float(&dummy);
                }
                ignore_white_space();
        }

        Mp++;

        return count;
}

//      Marks an integer list.
//      This is of the form ( i* )
//        where i is an integer.
//      If a specified string is found in the lookup and its value is 7, then the 7th value
//      in the array is set.
void mark_int_list(int *ilp, int max_ints, int lookup_type)
{
        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Marking integer list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;
        ignore_white_space();

        while (*Mp != ')') {
                if (*Mp == '"') {
                        int num = 0;
                        char str[128];

                        get_string(str);
                        switch(lookup_type) {
                                case SHIP_TYPE:
                                        num = ship_name_lookup(str);    // returns index of Ship[] entry with name
                                        break;

                                case SHIP_INFO_TYPE:
                                        num = ship_info_lookup(str);    // returns index of Ship_info[] entry with name
                                        break;

                                case WEAPON_LIST_TYPE:
                                        num = weapon_info_lookup(str);
                                        break;

                                default:
                                        Error(LOCATION,"Unknown lookup_type in stuff_int_list");
                                        break;
                        }

                        if ( (num < 0) || (num >= max_ints) )
                                Error(LOCATION, "Unable to find string \"%s\" in mark_int_list.\n", str);

//                      ilp[num] = 1;

                } else {
                        int     tval;

                        stuff_int(&tval);
                        Assert((tval >= 0) && (tval < max_ints));
                        if (tval >= 0 && tval < max_ints) {
                                ilp[tval] = 1;
                        }
                }

                ignore_white_space();
        }

        Mp++;

}


//      Stuff a vec3d struct, which is 3 floats.
void stuff_vec3d(vec3d *vp)
{
        stuff_float(&vp->xyz.x);
        stuff_float(&vp->xyz.y);
        stuff_float(&vp->xyz.z);
}

void stuff_parenthesized_vec3d(vec3d *vp)
{
        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading parenthesized vec3d.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        } else {
                Mp++;
                stuff_vec3d(vp);
                ignore_white_space();
                if (*Mp != ')') {
            error_display(1, "Reading parenthesized vec3d.  Found [%c].  Expecting ')'.\n", *Mp);
            throw parse::ParseException("Syntax error");
                }
                Mp++;
        }

}

//      Stuffs vec3d list.  *vlp is an array of vec3ds.
//      This is of the form ( (vec3d)* )
//        (where * is a kleene star, not a pointer indirection)
// For example, ( (1 2 3) (2 3 4) (2 3 5) )
//               is a list of three vec3ds.
int stuff_vec3d_list(vec3d *vlp, int max_vecs)
{
        int     count = 0;

        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading vec3d list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;

        ignore_white_space();

        while (*Mp != ')') {
                Assert(count < max_vecs);
                if (count < max_vecs) {
                        stuff_parenthesized_vec3d(&vlp[count++]);
                } else {
                        vec3d temp;
                        stuff_parenthesized_vec3d(&temp);
                }

                ignore_white_space();
        }

        Mp++;

        return count;
}

// ditto the above, but a vector of vec3ds...
int stuff_vec3d_list(SCP_vector<vec3d> &vec_list)
{
        ignore_white_space();

        if (*Mp != '(') {
        error_display(1, "Reading vec3d list.  Found [%c].  Expecting '('.\n", *Mp);
        throw parse::ParseException("Syntax error");
        }

        Mp++;

        ignore_white_space();

        while (*Mp != ')') {
                vec3d temp;
                stuff_parenthesized_vec3d(&temp);
                vec_list.push_back(temp);

                ignore_white_space();
        }

        Mp++;

        return vec_list.size();
}

//      Stuff a matrix, which is 3 vec3ds.
void stuff_matrix(matrix *mp)
{
        stuff_vec3d(&mp->vec.rvec);
        stuff_vec3d(&mp->vec.uvec);
        stuff_vec3d(&mp->vec.fvec);
}


//      Given a string, find it in a string array.
//      *description is only used for diagnostics in case it can't be found.
//      *str1 is the string to be found.
//      *strlist is the list of strings to search.
//      max is the number of entries in *strlist to scan.
int string_lookup(char *str1, char *strlist[], int max, char *description, int say_errors)
{
        int     i;

        for (i=0; i<max; i++) {
                Assert(strlen(strlist[i]) != 0); //-V805

                if (!stricmp(str1, strlist[i]))
                        return i;
        }

        if (say_errors)
                error_display(0, "Unable to find [%s] in %s list.\n", str1, description);

        return -1;
}

//      Find a required string (*id), then stuff the text of type f_type that
// follows it at *addr.  *strlist[] contains the strings it should try to
// match.
void find_and_stuff(char *id, int *addr, int f_type, char *strlist[], int max, char *description)
{
        char    token[128];
        int checking_ship_classes = (stricmp(id, "$class:") == 0);

        // Goober5000 - don't say errors when we're checking classes because 1) we have more checking to do; and 2) we will say a redundant error later
        required_string(id);
        stuff_string(token, f_type, sizeof(token));
        *addr = string_lookup(token, strlist, max, description, !checking_ship_classes);

        // Goober5000 - handle certain FSPort idiosyncracies with ship classes
        if (*addr < 0 && checking_ship_classes)
        {
                int idx = ship_info_lookup(token);

                if (idx >= 0)
                        *addr = string_lookup(Ship_info[idx].name, strlist, max, description, 0);
                else
                        *addr = -1;
        }
}

void find_and_stuff_optional(char *id, int *addr, int f_type, char *strlist[], int max, char *description)
{
        char token[128];

        if(optional_string(id))
        {
                stuff_string(token, f_type, sizeof(token));
                *addr = string_lookup(token, strlist, max, description, 1);
        }
}

//      Mp points at a string.
//      Find the string in the list of strings *strlist[].
// Returns the index of the match, -1 if none.
int match_and_stuff(int f_type, char *strlist[], int max, char *description)
{
        char    token[128];

        stuff_string(token, f_type, sizeof(token));
        return string_lookup(token, strlist, max, description, 0);
}

void find_and_stuff_or_add(char *id, int *addr, int f_type, char *strlist[], int *total,
        int max, char *description)
{
        char    token[128];

        *addr = -1;
        required_string(id);
        stuff_string(token, f_type, sizeof(token));
        if (*total)
                *addr = string_lookup(token, strlist, *total, description, 0);

        if (*addr == -1)  // not in list, so lets try and add it.
        {
                Assert(*total < max);
                strcpy(strlist[*total], token);
                *addr = (*total)++;
        }
}

// pause current parsing so that some else can be parsed without interfering
// with the currently parsing file
void pause_parse()
{
        Assert( !Parsing_paused );
        if (Parsing_paused)
                return;

        Mp_save = Mp;

        Warning_count_save = Warning_count;
        Error_count_save = Error_count;

        strcpy_s(Current_filename_save, Current_filename);

        Parsing_paused = 1;
}

// unpause parsing to continue with previously parsing file
void unpause_parse()
{
        Assert( Parsing_paused );
        if (!Parsing_paused)
                return;

        Mp = Mp_save;

        Warning_count = Warning_count_save;
        Error_count = Error_count_save;

        strcpy_s(Current_filename, Current_filename_save);

        Parsing_paused = 0;
}

void reset_parse(char *text)
{
        if (text != NULL) {
                Mp = text;
        } else {
                Mp = Mission_text;
        }

        Warning_count = 0;
        Error_count = 0;

        strcpy_s(Current_filename, Current_filename_sub);
}

// Display number of warnings and errors at the end of a parse.
void display_parse_diagnostics()
{
        nprintf(("Parse", "\nParse complete.\n"));
        nprintf(("Parse", "%i errors.  %i warnings.\n", Error_count, Warning_count));
}

// Splits a string into 2 lines if the string is wider than max_pixel_w pixels.  A null
// terminator is placed where required to make the first line <= max_pixel_w.  The remaining
// text is returned (leading whitespace removed).  If the line doesn't need to be split,
// NULL is returned.
char *split_str_once(char *src, int max_pixel_w)
{
        char *brk = NULL;
        int i, w, len, last_was_white = 0;

        Assert(src);
        Assert(max_pixel_w > 0);

        gr_get_string_size(&w, NULL, src);
        if ( (w <= max_pixel_w) && !strstr(src, "\n") ) {
                return NULL;  // string doesn't require a cut
        }

        len = strlen(src);
        for (i=0; i<len; i++) {
                gr_get_string_size(&w, NULL, src, i);
                if ( w > max_pixel_w )
                        break;

                if (src[i] == '\n') {  // reached natural end of line
                        src[i] = 0;
                        return src + i + 1;
                }

                if (is_white_space(src[i])) {
                        if (!last_was_white)
                                brk = src + i;

                        last_was_white = 1;

                } else {
                        last_was_white = 0;
                }
        }

        // if we are over max pixel width and weren't able to come up with a good non-word
        // split then just return the original src text and the calling function should
        // have to handle the result
        if ( (w > max_pixel_w) && ((i == 0) || !brk) ) {
                return src;
        }

        if (!brk) {
                brk = src + i;
        }

        *brk = 0;
        src = brk + 1;
        while (is_white_space(*src))
                src++;

        if (!*src)
                return NULL;  // end of the string anyway

        if (*src == '\n')
                src++;

        return src;
}

#define SPLIT_STR_BUFFER_SIZE   512

// --------------------------------------------------------------------------------------
// split_str()
//
// A general function that will split a string into several lines.  Lines are allowed up
// to max_pixel_w pixels.  Breaks are found in white space.
//
// Supports \n's in the strings!
//
// parameters:          src                     =>              source string to be broken up
//                                              max_pixel_w     =>              max width of line in pixels
//                                              n_chars         =>              output array that will hold number of characters in each line
//                                              p_str                   =>              output array of pointers to start of lines within src
//                                              max_lines       =>              limit of number of lines to break src up into
//                                              ignore_char     =>              OPTIONAL parameter (default val -1).  Ignore words starting with this character
//                                                                                              This is useful when you want to ignore embedded control information that starts
//                                                                                              with a specific character, like $ or #
//
//      returns:                        number of lines src is broken into
//                                              -1 is returned when an error occurs
//
int split_str(const char *src, int max_pixel_w, int *n_chars, const char **p_str, int max_lines, char ignore_char)
{
        char buffer[SPLIT_STR_BUFFER_SIZE];
        const char *breakpoint = NULL;
        int sw, new_line = 1, line_num = 0, last_was_white = 0;
        int ignore_until_whitespace, buf_index;

        // check our assumptions..
        Assert(src != NULL);
        Assert(n_chars != NULL);
        Assert(p_str != NULL);
        Assert(max_lines > 0);
        Assert(max_pixel_w > 0);

        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);
        buf_index = 0;
        ignore_until_whitespace = 0;

        // get rid of any leading whitespace
        while (is_white_space(*src))
                src++;

        new_line = 1;
        p_str[0] = NULL;

        // iterate through chars in line, keeping track of most recent "white space" location that can be used
        // as a line splitting point if necessary
        for (; *src; src++) {
                if (line_num >= max_lines)
                        return line_num;  // time to bail out

                // starting a new line of text, init stuff for that
                if (new_line) {
                        p_str[line_num] = NULL;
                        if (is_gray_space(*src))
                                continue;

                        p_str[line_num] = src;
                        breakpoint = NULL;
                        new_line = 0;
                }

                // maybe skip leading whitespace
                if (ignore_until_whitespace) {
                        if ( is_white_space(*src) )
                                ignore_until_whitespace = 0;

                        continue;
                }

                // if we have a newline, split the line here
                if (*src == '\n') {
                        n_chars[line_num] = src - p_str[line_num];  // track length of line
                        line_num++;
                        if (line_num < max_lines) {
                                p_str[line_num] = NULL;
                        }
                        new_line = 1;

                        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);
                        buf_index = 0;
                        continue;
                }

                if (*src == ignore_char) {
                        ignore_until_whitespace = 1;
                        continue;
                }

                if (is_gray_space(*src)) {
                        if (!last_was_white)  // track at first whitespace in a series of whitespace
                                breakpoint = src;

                        last_was_white = 1;

                } else {
                        // indicate next time around that this wasn't a whitespace character
                        last_was_white = 0;
                }

                Assertion(buf_index < SPLIT_STR_BUFFER_SIZE - 1, "buffer overflow in split_str: screen width causes this text to be longer than %d characters!", SPLIT_STR_BUFFER_SIZE - 1);

                // throw it in our buffer
                buffer[buf_index] = *src;
                buf_index++;
                buffer[buf_index] = 0;  // null terminate it

                gr_get_string_size(&sw, NULL, buffer);
                if (sw >= max_pixel_w) {
                        const char *end;

                        if (breakpoint) {
                                end = src = breakpoint;

                        } else {
                                end = src;  // force a split here since to whitespace
                                src--;  // reuse this character in next line
                        }

                        n_chars[line_num] = end - p_str[line_num];  // track length of line
                        Assert(n_chars[line_num]);
                        line_num++;
                        if (line_num < max_lines) {
                                p_str[line_num] = NULL;
                        }
                        new_line = 1;

                        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);
                        buf_index = 0;
                        continue;
                }
        }       // end for

        if (!new_line && p_str[line_num]) {
                n_chars[line_num] = src - p_str[line_num];  // track length of line
                Assert(n_chars[line_num]);
                line_num++;
        }

        return line_num;
}

int split_str(const char *src, int max_pixel_w, SCP_vector<int> &n_chars, SCP_vector<const char*> &p_str, char ignore_char)
{
        char buffer[SPLIT_STR_BUFFER_SIZE];
        const char *breakpoint = NULL;
        int sw, new_line = 1, line_num = 0, last_was_white = 0;
        int ignore_until_whitespace = 0, buf_index = 0;

        // check our assumptions..
        Assert(src != NULL);
        Assert(max_pixel_w > 0);

        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);

        // get rid of any leading whitespace
        while (is_white_space(*src))
                src++;

        p_str.clear();

        // iterate through chars in line, keeping track of most recent "white space" location that can be used
        // as a line splitting point if necessary
        for (; *src; src++) {

                // starting a new line of text, init stuff for that
                if (new_line) {
                        if (is_gray_space(*src))
                                continue;

                        p_str.push_back(src);
                        breakpoint = NULL;
                        new_line = 0;
                }

                // maybe skip leading whitespace
                if (ignore_until_whitespace) {
                        if ( is_white_space(*src) ) {
                                ignore_until_whitespace = 0;

                                // don't eat the newline
                                if (*src == EOLN)
                                        src--;
                        }

                        continue;
                }

                // if we have a newline, split the line here
                if (*src == '\n') {
                        n_chars.push_back(src - p_str.at(line_num));  // track length of line
                        line_num++;
                        new_line = 1;

                        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);
                        buf_index = 0;
                        continue;
                }

                if (*src == ignore_char) {
                        ignore_until_whitespace = 1;
                        continue;
                }

                if (is_gray_space(*src)) {
                        if (!last_was_white)  // track at first whitespace in a series of whitespace
                                breakpoint = src;

                        last_was_white = 1;

                } else {
                        // indicate next time around that this wasn't a whitespace character
                        last_was_white = 0;
                }

                Assertion(buf_index < SPLIT_STR_BUFFER_SIZE - 1, "buffer overflow in split_str: screen width causes this text to be longer than %d characters!", SPLIT_STR_BUFFER_SIZE - 1);

                // throw it in our buffer
                buffer[buf_index] = *src;
                buf_index++;
                buffer[buf_index] = 0;  // null terminate it

                gr_get_string_size(&sw, NULL, buffer);
                if (sw >= max_pixel_w) {
                        const char *end;

                        if (breakpoint) {
                                end = src = breakpoint;

                        } else {
                                end = src;  // force a split here since to whitespace
                                src--;  // reuse this character in next line
                        }

                        n_chars.push_back(end - p_str.at(line_num));  // track length of line
                        Assert(n_chars.at(line_num));
                        line_num++;
                        new_line = 1;

                        memset(buffer, 0, SPLIT_STR_BUFFER_SIZE);
                        buf_index = 0;
                        continue;
                }
        }       // end for

        if (!new_line && p_str.at(line_num)) {
                n_chars.push_back(src - p_str.at(line_num));  // track length of line
                Assert(n_chars.at(line_num));
                line_num++;
        }

        return line_num;
}

// Goober5000
// accounts for the dumb communications != communication, etc.
int subsystem_stricmp(const char *str1, const char *str2)
{
        Assert(str1 && str2);

        // ensure len-1 will be valid
        if (!*str1 || !*str2)
                return stricmp(str1, str2);

        // calc lengths
        int len1 = strlen(str1);
        int len2 = strlen(str2);

        // get rid of trailing s on s1?
        if (tolower(*(str1+len1-1) == 's'))
                len1--;

        // get rid of trailing s on s2?
        if (tolower(*(str2+len2-1) == 's'))
                len2--;

        // once we remove the trailing s on both names, they should be the same length
        if (len1 > len2)
                return 1;
        if (len1 < len2)
                return -1;

        // now do the comparison
        return strnicmp(str1, str2, len1);
}

// Goober5000
// current algorithm adapted from http://www.codeproject.com/string/stringsearch.asp
const char *stristr(const char *str, const char *substr)
{
        // check for null and insanity
        Assert(str);
        Assert(substr);
        if (str == NULL || substr == NULL || *substr == '\0')
                return NULL;

        // save both a lowercase and an uppercase version of the first character of substr
        char substr_ch_lower = (char)tolower(*substr);
        char substr_ch_upper = (char)toupper(*substr);

        // find the maximum distance to search
        const char *upper_bound = str + strlen(str) - strlen(substr);

        // loop through every character of str
        for (const char *start = str; start <= upper_bound; start++)
        {
                // check first character of substr
                if ((*start == substr_ch_upper) || (*start == substr_ch_lower))
                {
                        // first character matched, so check the rest
                        for (const char *str_ch = start+1, *substr_ch = substr+1; *substr_ch != '\0'; str_ch++, substr_ch++)
                        {
                                // character match?
                                if (*str_ch == *substr_ch)
                                        continue;

                                // converted character match?
                                if (tolower(*str_ch) == tolower(*substr_ch))
                                        continue;

                                // mismatch
                                goto stristr_continue_outer_loop;
                        }

                        // finished inner loop with success!
                        return start;
                }

stristr_continue_outer_loop:
                /* NO-OP */ ;
        }

        // no match
        return NULL;
}

// non-const version
char *stristr(char *str, const char *substr)
{
        // check for null and insanity
        Assert(str);
        Assert(substr);
        if (str == NULL || substr == NULL || *substr == '\0')
                return NULL;

        // save both a lowercase and an uppercase version of the first character of substr
        char substr_ch_lower = (char)tolower(*substr);
        char substr_ch_upper = (char)toupper(*substr);

        // find the maximum distance to search
        const char *upper_bound = str + strlen(str) - strlen(substr);

        // loop through every character of str
        for (char *start = str; start <= upper_bound; start++)
        {
                // check first character of substr
                if ((*start == substr_ch_upper) || (*start == substr_ch_lower))
                {
                        // first character matched, so check the rest
                        for (const char *str_ch = start+1, *substr_ch = substr+1; *substr_ch != '\0'; str_ch++, substr_ch++)
                        {
                                // character match?
                                if (*str_ch == *substr_ch)
                                        continue;

                                // converted character match?
                                if (tolower(*str_ch) == tolower(*substr_ch))
                                        continue;

                                // mismatch
                                goto stristr_continue_outer_loop;
                        }

                        // finished inner loop with success!
                        return start;
                }

stristr_continue_outer_loop:
                /* NO-OP */ ;
        }

        // no match
        return NULL;
}

// Goober5000
bool can_construe_as_integer(const char *text)
{
        // trivial case; evaluates to 0
        if (*text == '\0')
                return true;

        // number sign or digit for first char
        if ((*text != '+') && (*text != '-') && !isdigit(*text))
                return false;

        // check digits for rest
        for (const char *p = text + 1; *p != '\0'; p++)
        {
                if (!isdigit(*p))
                        return false;
        }

        return true;
}

// Goober5000
// yoinked gratefully from dbugfile.cpp
void vsprintf(SCP_string &dest, const char *format, va_list ap)
{
        va_list copy;
        
#if defined(_MSC_VER) && _MSC_VER < 1800
        // Only Visual Studio >= 2013 supports va_copy
        // This isn't portable but should work for Visual Studio
        copy = ap;
#else
        va_copy(copy, ap);
#endif

        int needed_length = vsnprintf(nullptr, 0, format, copy);
        va_end(copy);

        if (needed_length < 0) {
                // Error
                return;
        }

        dest.resize(static_cast<size_t>(needed_length));
        vsnprintf(&dest[0], dest.size() + 1, format, ap);
}

void sprintf(SCP_string &dest, const char *format, ...)
{
        va_list args;
        va_start(args, format);
        vsprintf(dest, format, args);
        va_end(args);
}

// Goober5000
bool end_string_at_first_hash_symbol(char *src)
{
        char *p;
        Assert(src);

        p = get_pointer_to_first_hash_symbol(src);
        if (p)
        {
                while ((p != src) && (*(p-1) == ' '))
                        p--;

                *p = '\0';
                return true;
        }

        return false;
}

// Goober5000
bool end_string_at_first_hash_symbol(SCP_string &src)
{
        int index = get_index_of_first_hash_symbol(src);
        if (index >= 0)
        {
                while (index > 0 && src[index-1] == ' ')
                        index--;

                src.resize(index);
                return true;
        }

        return false;
}

// Goober5000
char *get_pointer_to_first_hash_symbol(char *src)
{
        Assert(src);
        return strchr(src, '#');
}

// Goober5000
const char *get_pointer_to_first_hash_symbol(const char *src)
{
        Assert(src);
        return strchr(src, '#');
}

// Goober5000
int get_index_of_first_hash_symbol(SCP_string &src)
{
        size_t pos = src.find('#');
        return (pos == SCP_string::npos) ? -1 : pos;
}

// Goober5000
int replace_one(char *str, char *oldstr, char *newstr, uint max_len, int range)
{
        Assert(str && oldstr && newstr);

        // search
        char *ch = stristr(str, oldstr);

        // found?
        if (ch)
        {
                // not found within bounds?
                if ((range > 0) && ((ch - str) > range))
                {
                        return 0;
                }

                // determine if replacement will exceed max len
                if (strlen(str) + strlen(newstr) - strlen(oldstr) > max_len)
                {
                        return -1;
                }

                // allocate temp string to hold extra stuff
                char *temp = (char *) vm_malloc(sizeof(char) * max_len);

                // ensure allocation was successful
                if (temp)
                {
                        // save remainder of string
                        strcpy_s(temp, sizeof(char)*max_len, ch + strlen(oldstr));

                        // replace
                        strcpy(ch, newstr);

                        // append rest of string
                        strcpy(ch + strlen(newstr), temp);
                }

                // free temp string
                vm_free(temp);
        }
        // not found
        else
        {
                return 0;
        }

        // return pos of replacement
        return (ch - str);
}

// Goober5000
int replace_all(char *str, char *oldstr, char *newstr, uint max_len, int range)
{
        int val, tally = 0;

        while ((val = replace_one(str, oldstr, newstr, max_len, range)) > 0)
        {
                tally++;

                // adjust range (if we have one), because the text length might have changed
                if (range) {
                        range += strlen(newstr) - strlen(oldstr);
                }
        }

        return (val < 0) ? val : tally;
}

SCP_string& replace_one(SCP_string& context, const SCP_string& from, const SCP_string& to)
{
        size_t foundHere;
        if ((foundHere = context.find(from, 0)) != SCP_string::npos)
        {
                context.replace(foundHere, from.length(), to);
        }
        return context;
}

SCP_string& replace_one(SCP_string& context, const char* from, const char* to)
{
        size_t foundHere;
        if ((foundHere = context.find(from, 0)) != SCP_string::npos)
        {
                context.replace(foundHere, strlen(from), to);
        }
        return context;
}

// http://www.cppreference.com/wiki/string/replace
SCP_string& replace_all(SCP_string& context, const SCP_string& from, const SCP_string& to)
{
        size_t from_len = from.length();
        size_t to_len = to.length();

        size_t lookHere = 0;
        size_t foundHere;
        while ((foundHere = context.find(from, lookHere)) != SCP_string::npos)
        {
                context.replace(foundHere, from_len, to);
                lookHere = foundHere + to_len;
        }
        return context;
}

// http://www.cppreference.com/wiki/string/replace
SCP_string& replace_all(SCP_string& context, const char* from, const char* to)
{
        size_t from_len = strlen(from);
        size_t to_len = strlen(to);

        size_t lookHere = 0;
        size_t foundHere;
        while ((foundHere = context.find(from, lookHere)) != SCP_string::npos)
        {
                context.replace(foundHere, from_len, to);
                lookHere = foundHere + to_len;
        }
        return context;
}

// WMC
// Compares two strings, ignoring (last) extension
// Returns 0 if equal, nonzero if not
int strextcmp(const char *s1, const char *s2)
{
        // sanity check
        Assert( (s1 != NULL) && (s2 != NULL) );

        // find last '.' in both strings
        char *s1_end = (char *)strrchr(s1, '.');
        char *s2_end = (char *)strrchr(s2, '.');

        // get length
        size_t s1_len, s2_len;

        if (s1_end != NULL)
                s1_len = (s1_end - s1);
        else
                s1_len = strlen(s1);

        if (s2_end != NULL)
                s2_len = (s2_end - s2);
        else
                s2_len = strlen(s2);

        // if the lengths aren't the same then it's deffinitely not the same name
        if (s2_len != s1_len)
                return 1;

        return strnicmp(s1, s2, s1_len);
}

// Goober5000
bool drop_extension(char *str)
{
        char *p = strrchr(str, '.');
        if (p != NULL)
        {
                *p = 0;
                return true;
        }

        return false;
}

// Goober5000
bool drop_extension(SCP_string &str)
{
        size_t pos = str.rfind('.');
        if (pos != SCP_string::npos)
        {
                str.resize(pos);
                return true;
        }

        return false;
}

//WMC
void backspace(char* src)
{
        Assert(src!= NULL);             //this would be bad

        char *dest = src;
        src++;

        while(*src != '\0') {
                *dest++ = *src++;
        }

        *dest = '\0';
}

// Goober5000
void format_integer_with_commas(char *buf, int integer, bool use_comma_with_four_digits)
{
        int old_pos, new_pos, triad_count;
        char backward_buf[32];

        // print an initial string of just the digits
        sprintf(buf, "%d", integer);

        // no commas needed?
        if ((integer < 1000) || (integer < 10000 && !use_comma_with_four_digits))
                return;

        // scan the string backwards, writing commas after every third digit
        new_pos = 0;
        triad_count = 0;
        for (old_pos = strlen(buf) - 1; old_pos >= 0; old_pos--)
        {
                backward_buf[new_pos] = buf[old_pos];
                new_pos++;
                triad_count++;

                if (triad_count == 3 && old_pos > 0)
                {
                        backward_buf[new_pos] = ',';
                        new_pos++;
                        triad_count = 0;
                }
        }
        backward_buf[new_pos] = '\0';

        // now reverse the string
        new_pos = 0;
        for (old_pos = strlen(backward_buf) - 1; old_pos >= 0; old_pos--)
        {
                buf[new_pos] = backward_buf[old_pos];
                new_pos++;
        }
        buf[new_pos] = '\0';
}

// Goober5000
// there's probably a better way to do this, but this way works and is clear and short
int scan_fso_version_string(const char *text, int *major, int *minor, int *build, int *revis)
{
        int val;

        val = sscanf(text, ";;FSO %i.%i.%i.%i;;", major, minor, build, revis);
        if (val == 4)
                return val;

        *revis = 0;
        val = sscanf(text, ";;FSO %i.%i.%i;;", major, minor, build);
        if (val == 3)
                return val;

        *build = *revis = 0;
        val = sscanf(text, ";;FSO %i.%i;;", major, minor);
        if (val == 2)
                return val;

        *minor = *build = *revis = 0;
        val = sscanf(text, ";;FSO %i;;", major);
        if (val == 1)
                return val;

        *major = *minor = *build = *revis = 0;
        return 0;
}

// Goober5000 - used for long Warnings, Errors, and FRED error messages with SEXPs
void truncate_message_lines(SCP_string &text, int num_allowed_lines)
{
        Assert(num_allowed_lines > 0);
        size_t find_from = 0;

        while (find_from < text.size())
        {
                if (num_allowed_lines <= 0)
                {
                        text.resize(find_from);
                        text.append("[...]");
                        break;
                }

                size_t pos = text.find('\n', find_from);
                if (pos == SCP_string::npos)
                        break;

                num_allowed_lines--;
                find_from = pos + 1;
        }
}

// Goober5000 - ugh, I can't see why they didn't just use stuff_*_list for these;
// the only differece is the lack of parentheses

// from aicode.cpp
// Stuff a list of floats at *plist.
void parse_float_list(float *plist, int size)
{
        int i;

        for (i=0; i<size; i++)
        {
                stuff_float(&plist[i]);
        }
}

// from aicode.cpp
// Stuff a list of ints at *plist.
void parse_int_list(int *ilist, int size)
{
        int i;

        for (i=0; i<size; i++)
        {
                stuff_int(&ilist[i]);
        }
}

// parse a modular table of type "name_check" and parse it using the specified function callback
int parse_modular_table(const char *name_check, void (*parse_callback)(const char *filename), int path_type, int sort_type)
{
        SCP_vector<SCP_string> tbl_file_names;
        int i, num_files = 0;

        if ( (name_check == NULL) || (parse_callback == NULL) || ((*name_check) != '*') ) {
                Assertion(false, "parse_modular_table() called with invalid arguments; get a coder!\n");
                return 0;
        }

        num_files = cf_get_file_list(tbl_file_names, path_type, name_check, sort_type);

        Parsing_modular_table = true;

        for (i = 0; i < num_files; i++){
                tbl_file_names[i] += ".tbm";
                mprintf(("TBM  =>  Starting parse of '%s' ...\n", tbl_file_names[i].c_str()));
                (*parse_callback)(tbl_file_names[i].c_str());
        }

        Parsing_modular_table = false;

        return num_files;
}