osllex.l

/* Copyright Contributors to the Open Shading Language project.
 * SPDX-License-Identifier: BSD-3-Clause
 * https://github.com/AcademySoftwareFoundation/OpenShadingLanguage
 */


/** Lexical scanner for Open Shading Language
 **/

%top{
/* Include this first to avoid warnings about re-definitions on windows */
#include <stdint.h>
}

/************************************************************
 * Definitions section
 ************************************************************/

%option reentrant bison-bridge
%option extra-type="OSL::pvt::OSLCompilerImpl *"

/* Option 'noyywrap' indicates that when EOF is hit, yyin does not
 * automatically reset to another file.
 */
%option noyywrap
%option nounput noinput

/* never-interactive fixes a Windows compatibility problem where the
 * lexer emits isatty calls that don't exist.
 */
%option never-interactive

 /* Option 'prefix' creates a C++ lexer with the given prefix, so that
  * we can link with other flex-generated lexers in the same application
  * without name conflicts.
  */
%option prefix="osl"


 /* Define regular expression macros
  ************************************************/

 /* white space, not counting newline */
WHITE           [ \t\v\f\r]+
 /* alpha character */
ALPHA           [A-Za-z]
 /* numerals */
DIGIT           [0-9]
 /* Integer literal */
INTEGER         {DIGIT}+
HEXINTEGER      0[xX][0-9a-fA-F]+
 /* floating point literal (E, FLT1, FLT2, FLT3 are just helpers)
  * NB: we don't allow leading +/- due to ambiguity between
  * whether "a-0.5" is really "a -0.5" or "a - 0.5".  Resolve this
  * in the grammar.
  */
E               [eE][-+]?{DIGIT}+
FLT1            {DIGIT}+\.{DIGIT}*{E}?
FLT2            {DIGIT}*\.{DIGIT}+{E}?
FLT3            {DIGIT}+{E}
FLT             {FLT1}|{FLT2}|{FLT3}
 /* string literal */
STR     \"(\\.|[^\\"\n])*\"
        /* " This extra quote fixes emacs syntax highlighting on this file */
 /* Identifier: alphanumeric, may contain digits after the first character */
IDENT           ({ALPHA}|[_])({ALPHA}|{DIGIT}|[_])*
 /* C preprocessor (cpp) directives */
CPP             ^[ \t]*#.*\n
CPLUSCOMMENT    \/\/.*\n



 /* Note for lex newbies: the following '%{ .. %}' section contains literal
  * C code that will be inserted at the top of code that flex generates.
  */
%{
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <vector>
#include <string>

#include <OpenImageIO/thread.h>
#include <OpenImageIO/strutil.h>
#include "oslcomp_pvt.h"

using namespace OSL;
using namespace OSL::pvt;

#include "oslgram.hpp"   /* Generated by bison/yacc */


#ifdef _WIN32
#define YY_NO_UNISTD_H

#ifdef _MSC_VER
#define strtoll _strtoi64
#include <io.h>
#define isatty _isatty
#endif
#endif

// flex itself will generate fatal warnings about signed vs unsigned.
// Bypass that nonsense.
#if OSL_GNUC_VERSION >= 60000
#pragma GCC diagnostic ignored "-Wsign-compare"
#endif

// flex uses the 'register' keyword, warned because it's deprecated in C++17.
#if defined(__clang__)
#pragma GCC diagnostic ignored "-Wdeprecated-register"
#pragma GCC diagnostic ignored "-Wregister"
#endif
#if OSL_GNUC_VERSION >= 90000
#pragma GCC diagnostic ignored "-Wregister"
#endif

#if defined(__GNUC__) || defined(__clang__)
#pragma GCC visibility push(hidden)
#endif

OSL_NAMESPACE_BEGIN
namespace pvt {
    void preprocess (const char *str, OSLCompilerImpl* oslcompiler, YYLTYPE* yyloc_param);
    int osllex(YYSTYPE* yylval_param, YYLTYPE* yyloc_param, void* yyscanner, OSLCompilerImpl* oslcompiler);
}
OSL_NAMESPACE_END
#define YY_DECL int OSL::pvt::osllex(YYSTYPE* yylval_param, YYLTYPE* yyloc_param, void* yyscanner, OSLCompilerImpl* oslcompiler)

// Macro that sets the yylloc line variables to the current parse line.
#define SETLINE yyloc_param->first_line = yyloc_param->last_line = oslcompiler->lineno()

%}

%%

 /************************************************
  * Lexical matching rules
  ************************************************/

 /* preprocessor symbols */
{CPP}                   {  OSL::pvt::preprocess (yytext, oslcompiler, yyloc_param); SETLINE; }

 /* Comments */
{CPLUSCOMMENT}          {  oslcompiler->incr_lineno(); /* skip it */
                           SETLINE;
                        }

 /* keywords */
"break"                 {  SETLINE;  return (yylval->i=BREAK); }
"closure"               {  SETLINE;  return (yylval->i=CLOSURE); }
"color"                 {  SETLINE;  return (yylval->i=COLORTYPE); }
"continue"              {  SETLINE;  return (yylval->i=CONTINUE); }
"do"                    {  SETLINE;  return (yylval->i=DO); }
"else"                  {  SETLINE;  return (yylval->i=ELSE); }
"float"                 {  SETLINE;  return (yylval->i=FLOATTYPE); }
"for"                   {  SETLINE;  return (yylval->i=FOR); }
"if"                    {  SETLINE;  return (yylval->i=IF_TOKEN); }
"illuminance"           {  SETLINE;  return (yylval->i=ILLUMINANCE); }
"illuminate"            {  SETLINE;  return (yylval->i=ILLUMINATE); }
"int"                   {  SETLINE;  return (yylval->i=INTTYPE); }
"matrix"                {  SETLINE;  return (yylval->i=MATRIXTYPE); }
"normal"                {  SETLINE;  return (yylval->i=NORMALTYPE); }
"output"                {  SETLINE;  return (yylval->i=OUTPUT); }
"point"                 {  SETLINE;  return (yylval->i=POINTTYPE); }
"public"                {  SETLINE;  return (yylval->i=PUBLIC); }
"return"                {  SETLINE;  return (yylval->i=RETURN); }
"string"                {  SETLINE;  return (yylval->i=STRINGTYPE); }
"struct"                {  SETLINE;  return (yylval->i=STRUCT); }
"vector"                {  SETLINE;  return (yylval->i=VECTORTYPE); }
"void"                  {  SETLINE;  return (yylval->i=VOIDTYPE); }
"while"                 {  SETLINE;  return (yylval->i=WHILE); }
"or"                    {  SETLINE;  return (yylval->i=OR_OP); }
"and"                   {  SETLINE;  return (yylval->i=AND_OP); }
"not"                   {  SETLINE;  return (yylval->i=NOT_OP); }

 /* reserved words */
"bool"|"case"|"char"|"class"|"const"|"default"|"double" |    \
"enum"|"extern"|"false"|"friend"|"inline"|"long"|"private" | \
"protected"|"short"|"signed"|"sizeof"|"static"|"struct" |    \
"switch"|"template"|"this"|"true"|"typedef"|"uniform" |      \
"union"|"unsigned"|"varying"|"virtual" {
                            oslcompiler->errorfmt(oslcompiler->filename(),
                                                  oslcompiler->lineno(),
                                                  "'{}' is a reserved word",
                                                  yytext);
                            SETLINE;
                            return (yylval->i=RESERVED);
                        }


 /* Identifiers */
{IDENT}                 {
                            yylval->s = ustring(yytext).c_str();
                            SETLINE;
                            return IDENTIFIER;
                        }

 /* Literal values */
{INTEGER}               {
                            long long llval = strtoll (yytext, (char**)NULL, 10);
                            // we do not detect overflow when the value is INT_MAX+1,
                            // because negation happens later and -(INT_MAX+1) == INT_MIN
                            if (llval > ((long long)INT_MAX)+1) {
                                oslcompiler->errorfmt(oslcompiler->filename(),
                                                      oslcompiler->lineno(),
                                                      "integer overflow, value must be between {} and {}.",
                                                      INT_MIN, INT_MAX);
                            }
                            yylval->i = (int)llval;
                            SETLINE;
                            return INT_LITERAL;
                        }
{HEXINTEGER}            {
                            long long llval = strtoll (yytext, (char**)NULL, 16);
                            // we do not detect overflow when the value is INT_MAX+1,
                            // because negation happens later and -(INT_MAX+1) == INT_MIN
                            if (llval > ((long long)UINT_MAX)+1) {
                                oslcompiler->errorfmt(oslcompiler->filename(),
                                                      oslcompiler->lineno(),
                                                      "integer overflow, value must be between {} and {}.",
                                                      INT_MIN, INT_MAX);
                            }
                            yylval->i = (int)llval;
                            SETLINE;
                            return INT_LITERAL;
                        }



{FLT}                   {
                            yylval->f = OIIO::Strutil::from_string<float>(yytext);
                            SETLINE;
                            return FLOAT_LITERAL;
                        }

{STR}                   {
                            // grab the material between the quotes
                            string_view s(yytext + 1, yyleng - 2);
                            std::string unescaped;
                            if (s.find('\\') != string_view::npos) {
                                // Only make a new string if we must unescape
                                unescaped = OIIO::Strutil::unescape_chars(s);
                                s = string_view(unescaped);
                            }
                            yylval->s = ustring(s).c_str();
                            SETLINE;
                            // std::cerr << "osllex string '" << yylval->s << "'\n";
                            return STRING_LITERAL;
                        }

 /* The one-char operators (like "+") will return correctly with the
  * catch-all rule, but we need to define the two-character operators
  * so they are not lexed as '+' and '=' separately, for example.
  */
"+="                    {  SETLINE;  return (yylval->i=ADD_ASSIGN); }
"-="                    {  SETLINE;  return (yylval->i=SUB_ASSIGN); }
"*="                    {  SETLINE;  return (yylval->i=MUL_ASSIGN); }
"/="                    {  SETLINE;  return (yylval->i=DIV_ASSIGN); }
"&="                    {  SETLINE;  return (yylval->i=BIT_AND_ASSIGN); }
"|="                    {  SETLINE;  return (yylval->i=BIT_OR_ASSIGN); }
"^="                    {  SETLINE;  return (yylval->i=XOR_ASSIGN); }
"<<="                   {  SETLINE;  return (yylval->i=SHL_ASSIGN); }
">>="                   {  SETLINE;  return (yylval->i=SHR_ASSIGN); }
"<<"                    {  SETLINE;  return (yylval->i=SHL_OP); }
">>"                    {  SETLINE;  return (yylval->i=SHR_OP); }
"&&"                    {  SETLINE;  return (yylval->i=AND_OP); }
"||"                    {  SETLINE;  return (yylval->i=OR_OP); }
"<="                    {  SETLINE;  return (yylval->i=LE_OP); }
">="                    {  SETLINE;  return (yylval->i=GE_OP); }
"=="                    {  SETLINE;  return (yylval->i=EQ_OP); }
"!="                    {  SETLINE;  return (yylval->i=NE_OP); }
"++"                    {  SETLINE;  return (yylval->i=INCREMENT); }
"--"                    {  SETLINE;  return (yylval->i=DECREMENT); }

 /* Beginning of metadata */
"[["                    {  SETLINE;  return (yylval->i=METADATA_BEGIN); }

 /* End of line */
"\\\n"                  |
[\n]                    {  oslcompiler->incr_lineno();
                           SETLINE;
                        }

 /* Ignore whitespace */
{WHITE}                 {  }

 /* catch-all rule for any other single characters */
!                       {  SETLINE;  return (yylval->i = NOT_OP); }
.                       {  SETLINE;  return (yylval->i = *yytext); }

%%

OSL_NAMESPACE_BEGIN
namespace pvt {   // OSL::pvt

void
preprocess (const char *str, OSLCompilerImpl* oslcompiler, YYLTYPE* yyloc_param)
{
#if 0
    printf ("preprocess: <%s>\n", str);
#endif
    const char *p = str;
    while (*p == ' ' || *p == '\t')
        p++;
    if (*p != '#') {
        oslcompiler->errorfmt(oslcompiler->filename(), oslcompiler->lineno(),
                              "Possible bug in shader preprocess");
        SETLINE;
        return;
    }
    p++;
    while (*p == ' ' || *p == '\t')
        p++;
    if (! strncmp (p, "pragma", 6)) {
        // pragma
        OIIO::string_view line (p+6);
        string_view pragmatype = OIIO::Strutil::parse_word (line);
        if (pragmatype == "error") {
            string_view msg;
            if (OIIO::Strutil::parse_string(line, msg))
                oslcompiler->errorfmt(oslcompiler->filename(),
                                      oslcompiler->lineno(), "{}", msg);
            else
                oslcompiler->errorfmt(oslcompiler->filename(),
                                      oslcompiler->lineno(),
                                      "<unknown error>");
        }
        else if (pragmatype == "warning") {
            string_view msg;
            if (OIIO::Strutil::parse_string(line, msg))
                oslcompiler->warningfmt(oslcompiler->filename(),
                                      oslcompiler->lineno(), "{}", msg);
            else
                oslcompiler->warningfmt(oslcompiler->filename(),
                                        oslcompiler->lineno(),
                                        "<unknown warning>");
        }
        else if (OIIO::Strutil::iequals (pragmatype, "osl")) {
            string_view pragmaname = OIIO::Strutil::parse_word (line);
            if (pragmaname == "nowarn") {
                oslcompiler->pragma_nowarn ();
            } else {
                oslcompiler->warningfmt(oslcompiler->filename(), oslcompiler->lineno(),
                                        "Unknown pragma '{}'", pragmaname);
            }
        }
        // N.B. Any other pragmas that don't start with "osl" are ignored
        oslcompiler->incr_lineno();  // the pragma ends with an EOLN
    } else {  /* probably the line number and filename */
        if (! strncmp (p, "line", 4))
            p += 4;
        int line = atoi (p);
        if (line > 0) {
            const char *f = strchr (str, '\"'); // " undo syntax highlight
            if (f) {
                ++f;  // increment to past the quote
                int len = 0;  // count of chars within quotes
                while (f[len] && f[len] != '\"') // " undo syntax highlight
                    ++len;
                std::string filename (f, len);
                // Make it relative to the working directory, so error output
                // isn't hugely cluttered (and also to make the preprocessor's
                // error output match the standard cpp.
                if (filename.find (oslcompiler->cwd()) == 0) {
                    filename.erase (0, oslcompiler->cwd().size());
                    if (filename.size() &&
                        (filename[0] == '/' || filename[0] == '\\'))
                        filename.erase (0, 1);
                }
                ustring ufilename(filename);
                oslcompiler->filename (ufilename);
                // TODO: We no longer use Boost Wave, so verify if the
                // workaround below is still needed ..
                // Spooky workaround for Boost Wave bug: force_include
                // is broken and doesn't give us the right lines/files, so
                // instead we forcefully insert a '#include "stdosl.h"' into
                // the stream ourselves, but this in turn makes the rest
                // of the main file all have line counts one line off!
                // So we fix it here, ugh.
                if (ufilename == oslcompiler->main_filename())
                    --line;
            }
            oslcompiler->lineno (line);
        } else {
            oslcompiler->errorfmt(oslcompiler->filename(), oslcompiler->lineno(),
                                  "Unrecognized preprocessor command: #{}", p);
        }
    }
    SETLINE;
}

static std::mutex oslcompiler_mutex;

bool
OSLCompilerImpl::osl_parse_buffer (const std::string &preprocessed_buffer)
{
    // Thread safety with the lexer/parser
    std::lock_guard<std::mutex> lock(oslcompiler_mutex);

    yyscan_t scanner;
    yylex_init(&scanner);
    YY_BUFFER_STATE buffer = yy_scan_string(preprocessed_buffer.c_str(), scanner);
    yy_switch_to_buffer (buffer, scanner);
    oslparse (scanner, this);
    bool parseerr = error_encountered();
    yy_delete_buffer (buffer, scanner);
    yylex_destroy(scanner);

    return parseerr;
}

}; // namespace pvt
OSL_NAMESPACE_END

#if defined(__GNUC__) || defined(__clang__)
#pragma GCC visibility pop
#endif