9bbeb35c08
Fixed an issue that prevented the password reset tokens from working. Added email templates for password reset success and new account creation. Added more dynamic email template support.
1015 lines
44 KiB
Python
1015 lines
44 KiB
Python
from . import model
|
|
from .commontypes import COMMON_TYPES, resolve_common_type
|
|
from .error import FFIError, CDefError
|
|
try:
|
|
from . import _pycparser as pycparser
|
|
except ImportError:
|
|
import pycparser
|
|
import weakref, re, sys
|
|
|
|
try:
|
|
if sys.version_info < (3,):
|
|
import thread as _thread
|
|
else:
|
|
import _thread
|
|
lock = _thread.allocate_lock()
|
|
except ImportError:
|
|
lock = None
|
|
|
|
def _workaround_for_static_import_finders():
|
|
# Issue #392: packaging tools like cx_Freeze can not find these
|
|
# because pycparser uses exec dynamic import. This is an obscure
|
|
# workaround. This function is never called.
|
|
import pycparser.yacctab
|
|
import pycparser.lextab
|
|
|
|
CDEF_SOURCE_STRING = "<cdef source string>"
|
|
_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
|
|
re.DOTALL | re.MULTILINE)
|
|
_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
|
|
r"\b((?:[^\n\\]|\\.)*?)$",
|
|
re.DOTALL | re.MULTILINE)
|
|
_r_line_directive = re.compile(r"^[ \t]*#[ \t]*(?:line|\d+)\b.*$", re.MULTILINE)
|
|
_r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}")
|
|
_r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$")
|
|
_r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
|
|
_r_words = re.compile(r"\w+|\S")
|
|
_parser_cache = None
|
|
_r_int_literal = re.compile(r"-?0?x?[0-9a-f]+[lu]*$", re.IGNORECASE)
|
|
_r_stdcall1 = re.compile(r"\b(__stdcall|WINAPI)\b")
|
|
_r_stdcall2 = re.compile(r"[(]\s*(__stdcall|WINAPI)\b")
|
|
_r_cdecl = re.compile(r"\b__cdecl\b")
|
|
_r_extern_python = re.compile(r'\bextern\s*"'
|
|
r'(Python|Python\s*\+\s*C|C\s*\+\s*Python)"\s*.')
|
|
_r_star_const_space = re.compile( # matches "* const "
|
|
r"[*]\s*((const|volatile|restrict)\b\s*)+")
|
|
_r_int_dotdotdot = re.compile(r"(\b(int|long|short|signed|unsigned|char)\s*)+"
|
|
r"\.\.\.")
|
|
_r_float_dotdotdot = re.compile(r"\b(double|float)\s*\.\.\.")
|
|
|
|
def _get_parser():
|
|
global _parser_cache
|
|
if _parser_cache is None:
|
|
_parser_cache = pycparser.CParser()
|
|
return _parser_cache
|
|
|
|
def _workaround_for_old_pycparser(csource):
|
|
# Workaround for a pycparser issue (fixed between pycparser 2.10 and
|
|
# 2.14): "char*const***" gives us a wrong syntax tree, the same as
|
|
# for "char***(*const)". This means we can't tell the difference
|
|
# afterwards. But "char(*const(***))" gives us the right syntax
|
|
# tree. The issue only occurs if there are several stars in
|
|
# sequence with no parenthesis inbetween, just possibly qualifiers.
|
|
# Attempt to fix it by adding some parentheses in the source: each
|
|
# time we see "* const" or "* const *", we add an opening
|
|
# parenthesis before each star---the hard part is figuring out where
|
|
# to close them.
|
|
parts = []
|
|
while True:
|
|
match = _r_star_const_space.search(csource)
|
|
if not match:
|
|
break
|
|
#print repr(''.join(parts)+csource), '=>',
|
|
parts.append(csource[:match.start()])
|
|
parts.append('('); closing = ')'
|
|
parts.append(match.group()) # e.g. "* const "
|
|
endpos = match.end()
|
|
if csource.startswith('*', endpos):
|
|
parts.append('('); closing += ')'
|
|
level = 0
|
|
i = endpos
|
|
while i < len(csource):
|
|
c = csource[i]
|
|
if c == '(':
|
|
level += 1
|
|
elif c == ')':
|
|
if level == 0:
|
|
break
|
|
level -= 1
|
|
elif c in ',;=':
|
|
if level == 0:
|
|
break
|
|
i += 1
|
|
csource = csource[endpos:i] + closing + csource[i:]
|
|
#print repr(''.join(parts)+csource)
|
|
parts.append(csource)
|
|
return ''.join(parts)
|
|
|
|
def _preprocess_extern_python(csource):
|
|
# input: `extern "Python" int foo(int);` or
|
|
# `extern "Python" { int foo(int); }`
|
|
# output:
|
|
# void __cffi_extern_python_start;
|
|
# int foo(int);
|
|
# void __cffi_extern_python_stop;
|
|
#
|
|
# input: `extern "Python+C" int foo(int);`
|
|
# output:
|
|
# void __cffi_extern_python_plus_c_start;
|
|
# int foo(int);
|
|
# void __cffi_extern_python_stop;
|
|
parts = []
|
|
while True:
|
|
match = _r_extern_python.search(csource)
|
|
if not match:
|
|
break
|
|
endpos = match.end() - 1
|
|
#print
|
|
#print ''.join(parts)+csource
|
|
#print '=>'
|
|
parts.append(csource[:match.start()])
|
|
if 'C' in match.group(1):
|
|
parts.append('void __cffi_extern_python_plus_c_start; ')
|
|
else:
|
|
parts.append('void __cffi_extern_python_start; ')
|
|
if csource[endpos] == '{':
|
|
# grouping variant
|
|
closing = csource.find('}', endpos)
|
|
if closing < 0:
|
|
raise CDefError("'extern \"Python\" {': no '}' found")
|
|
if csource.find('{', endpos + 1, closing) >= 0:
|
|
raise NotImplementedError("cannot use { } inside a block "
|
|
"'extern \"Python\" { ... }'")
|
|
parts.append(csource[endpos+1:closing])
|
|
csource = csource[closing+1:]
|
|
else:
|
|
# non-grouping variant
|
|
semicolon = csource.find(';', endpos)
|
|
if semicolon < 0:
|
|
raise CDefError("'extern \"Python\": no ';' found")
|
|
parts.append(csource[endpos:semicolon+1])
|
|
csource = csource[semicolon+1:]
|
|
parts.append(' void __cffi_extern_python_stop;')
|
|
#print ''.join(parts)+csource
|
|
#print
|
|
parts.append(csource)
|
|
return ''.join(parts)
|
|
|
|
def _warn_for_string_literal(csource):
|
|
if '"' not in csource:
|
|
return
|
|
for line in csource.splitlines():
|
|
if '"' in line and not line.lstrip().startswith('#'):
|
|
import warnings
|
|
warnings.warn("String literal found in cdef() or type source. "
|
|
"String literals are ignored here, but you should "
|
|
"remove them anyway because some character sequences "
|
|
"confuse pre-parsing.")
|
|
break
|
|
|
|
def _warn_for_non_extern_non_static_global_variable(decl):
|
|
if not decl.storage:
|
|
import warnings
|
|
warnings.warn("Global variable '%s' in cdef(): for consistency "
|
|
"with C it should have a storage class specifier "
|
|
"(usually 'extern')" % (decl.name,))
|
|
|
|
def _remove_line_directives(csource):
|
|
# _r_line_directive matches whole lines, without the final \n, if they
|
|
# start with '#line' with some spacing allowed, or '#NUMBER'. This
|
|
# function stores them away and replaces them with exactly the string
|
|
# '#line@N', where N is the index in the list 'line_directives'.
|
|
line_directives = []
|
|
def replace(m):
|
|
i = len(line_directives)
|
|
line_directives.append(m.group())
|
|
return '#line@%d' % i
|
|
csource = _r_line_directive.sub(replace, csource)
|
|
return csource, line_directives
|
|
|
|
def _put_back_line_directives(csource, line_directives):
|
|
def replace(m):
|
|
s = m.group()
|
|
if not s.startswith('#line@'):
|
|
raise AssertionError("unexpected #line directive "
|
|
"(should have been processed and removed")
|
|
return line_directives[int(s[6:])]
|
|
return _r_line_directive.sub(replace, csource)
|
|
|
|
def _preprocess(csource):
|
|
# First, remove the lines of the form '#line N "filename"' because
|
|
# the "filename" part could confuse the rest
|
|
csource, line_directives = _remove_line_directives(csource)
|
|
# Remove comments. NOTE: this only work because the cdef() section
|
|
# should not contain any string literals (except in line directives)!
|
|
def replace_keeping_newlines(m):
|
|
return ' ' + m.group().count('\n') * '\n'
|
|
csource = _r_comment.sub(replace_keeping_newlines, csource)
|
|
# Remove the "#define FOO x" lines
|
|
macros = {}
|
|
for match in _r_define.finditer(csource):
|
|
macroname, macrovalue = match.groups()
|
|
macrovalue = macrovalue.replace('\\\n', '').strip()
|
|
macros[macroname] = macrovalue
|
|
csource = _r_define.sub('', csource)
|
|
#
|
|
if pycparser.__version__ < '2.14':
|
|
csource = _workaround_for_old_pycparser(csource)
|
|
#
|
|
# BIG HACK: replace WINAPI or __stdcall with "volatile const".
|
|
# It doesn't make sense for the return type of a function to be
|
|
# "volatile volatile const", so we abuse it to detect __stdcall...
|
|
# Hack number 2 is that "int(volatile *fptr)();" is not valid C
|
|
# syntax, so we place the "volatile" before the opening parenthesis.
|
|
csource = _r_stdcall2.sub(' volatile volatile const(', csource)
|
|
csource = _r_stdcall1.sub(' volatile volatile const ', csource)
|
|
csource = _r_cdecl.sub(' ', csource)
|
|
#
|
|
# Replace `extern "Python"` with start/end markers
|
|
csource = _preprocess_extern_python(csource)
|
|
#
|
|
# Now there should not be any string literal left; warn if we get one
|
|
_warn_for_string_literal(csource)
|
|
#
|
|
# Replace "[...]" with "[__dotdotdotarray__]"
|
|
csource = _r_partial_array.sub('[__dotdotdotarray__]', csource)
|
|
#
|
|
# Replace "...}" with "__dotdotdotNUM__}". This construction should
|
|
# occur only at the end of enums; at the end of structs we have "...;}"
|
|
# and at the end of vararg functions "...);". Also replace "=...[,}]"
|
|
# with ",__dotdotdotNUM__[,}]": this occurs in the enums too, when
|
|
# giving an unknown value.
|
|
matches = list(_r_partial_enum.finditer(csource))
|
|
for number, match in enumerate(reversed(matches)):
|
|
p = match.start()
|
|
if csource[p] == '=':
|
|
p2 = csource.find('...', p, match.end())
|
|
assert p2 > p
|
|
csource = '%s,__dotdotdot%d__ %s' % (csource[:p], number,
|
|
csource[p2+3:])
|
|
else:
|
|
assert csource[p:p+3] == '...'
|
|
csource = '%s __dotdotdot%d__ %s' % (csource[:p], number,
|
|
csource[p+3:])
|
|
# Replace "int ..." or "unsigned long int..." with "__dotdotdotint__"
|
|
csource = _r_int_dotdotdot.sub(' __dotdotdotint__ ', csource)
|
|
# Replace "float ..." or "double..." with "__dotdotdotfloat__"
|
|
csource = _r_float_dotdotdot.sub(' __dotdotdotfloat__ ', csource)
|
|
# Replace all remaining "..." with the same name, "__dotdotdot__",
|
|
# which is declared with a typedef for the purpose of C parsing.
|
|
csource = csource.replace('...', ' __dotdotdot__ ')
|
|
# Finally, put back the line directives
|
|
csource = _put_back_line_directives(csource, line_directives)
|
|
return csource, macros
|
|
|
|
def _common_type_names(csource):
|
|
# Look in the source for what looks like usages of types from the
|
|
# list of common types. A "usage" is approximated here as the
|
|
# appearance of the word, minus a "definition" of the type, which
|
|
# is the last word in a "typedef" statement. Approximative only
|
|
# but should be fine for all the common types.
|
|
look_for_words = set(COMMON_TYPES)
|
|
look_for_words.add(';')
|
|
look_for_words.add(',')
|
|
look_for_words.add('(')
|
|
look_for_words.add(')')
|
|
look_for_words.add('typedef')
|
|
words_used = set()
|
|
is_typedef = False
|
|
paren = 0
|
|
previous_word = ''
|
|
for word in _r_words.findall(csource):
|
|
if word in look_for_words:
|
|
if word == ';':
|
|
if is_typedef:
|
|
words_used.discard(previous_word)
|
|
look_for_words.discard(previous_word)
|
|
is_typedef = False
|
|
elif word == 'typedef':
|
|
is_typedef = True
|
|
paren = 0
|
|
elif word == '(':
|
|
paren += 1
|
|
elif word == ')':
|
|
paren -= 1
|
|
elif word == ',':
|
|
if is_typedef and paren == 0:
|
|
words_used.discard(previous_word)
|
|
look_for_words.discard(previous_word)
|
|
else: # word in COMMON_TYPES
|
|
words_used.add(word)
|
|
previous_word = word
|
|
return words_used
|
|
|
|
|
|
class Parser(object):
|
|
|
|
def __init__(self):
|
|
self._declarations = {}
|
|
self._included_declarations = set()
|
|
self._anonymous_counter = 0
|
|
self._structnode2type = weakref.WeakKeyDictionary()
|
|
self._options = {}
|
|
self._int_constants = {}
|
|
self._recomplete = []
|
|
self._uses_new_feature = None
|
|
|
|
def _parse(self, csource):
|
|
csource, macros = _preprocess(csource)
|
|
# XXX: for more efficiency we would need to poke into the
|
|
# internals of CParser... the following registers the
|
|
# typedefs, because their presence or absence influences the
|
|
# parsing itself (but what they are typedef'ed to plays no role)
|
|
ctn = _common_type_names(csource)
|
|
typenames = []
|
|
for name in sorted(self._declarations):
|
|
if name.startswith('typedef '):
|
|
name = name[8:]
|
|
typenames.append(name)
|
|
ctn.discard(name)
|
|
typenames += sorted(ctn)
|
|
#
|
|
csourcelines = []
|
|
csourcelines.append('# 1 "<cdef automatic initialization code>"')
|
|
for typename in typenames:
|
|
csourcelines.append('typedef int %s;' % typename)
|
|
csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
|
|
' __dotdotdot__;')
|
|
# this forces pycparser to consider the following in the file
|
|
# called <cdef source string> from line 1
|
|
csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
|
|
csourcelines.append(csource)
|
|
csourcelines.append('') # see test_missing_newline_bug
|
|
fullcsource = '\n'.join(csourcelines)
|
|
if lock is not None:
|
|
lock.acquire() # pycparser is not thread-safe...
|
|
try:
|
|
ast = _get_parser().parse(fullcsource)
|
|
except pycparser.c_parser.ParseError as e:
|
|
self.convert_pycparser_error(e, csource)
|
|
finally:
|
|
if lock is not None:
|
|
lock.release()
|
|
# csource will be used to find buggy source text
|
|
return ast, macros, csource
|
|
|
|
def _convert_pycparser_error(self, e, csource):
|
|
# xxx look for "<cdef source string>:NUM:" at the start of str(e)
|
|
# and interpret that as a line number. This will not work if
|
|
# the user gives explicit ``# NUM "FILE"`` directives.
|
|
line = None
|
|
msg = str(e)
|
|
match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
|
|
if match:
|
|
linenum = int(match.group(1), 10)
|
|
csourcelines = csource.splitlines()
|
|
if 1 <= linenum <= len(csourcelines):
|
|
line = csourcelines[linenum-1]
|
|
return line
|
|
|
|
def convert_pycparser_error(self, e, csource):
|
|
line = self._convert_pycparser_error(e, csource)
|
|
|
|
msg = str(e)
|
|
if line:
|
|
msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
|
|
else:
|
|
msg = 'parse error\n%s' % (msg,)
|
|
raise CDefError(msg)
|
|
|
|
def parse(self, csource, override=False, packed=False, pack=None,
|
|
dllexport=False):
|
|
if packed:
|
|
if packed != True:
|
|
raise ValueError("'packed' should be False or True; use "
|
|
"'pack' to give another value")
|
|
if pack:
|
|
raise ValueError("cannot give both 'pack' and 'packed'")
|
|
pack = 1
|
|
elif pack:
|
|
if pack & (pack - 1):
|
|
raise ValueError("'pack' must be a power of two, not %r" %
|
|
(pack,))
|
|
else:
|
|
pack = 0
|
|
prev_options = self._options
|
|
try:
|
|
self._options = {'override': override,
|
|
'packed': pack,
|
|
'dllexport': dllexport}
|
|
self._internal_parse(csource)
|
|
finally:
|
|
self._options = prev_options
|
|
|
|
def _internal_parse(self, csource):
|
|
ast, macros, csource = self._parse(csource)
|
|
# add the macros
|
|
self._process_macros(macros)
|
|
# find the first "__dotdotdot__" and use that as a separator
|
|
# between the repeated typedefs and the real csource
|
|
iterator = iter(ast.ext)
|
|
for decl in iterator:
|
|
if decl.name == '__dotdotdot__':
|
|
break
|
|
else:
|
|
assert 0
|
|
current_decl = None
|
|
#
|
|
try:
|
|
self._inside_extern_python = '__cffi_extern_python_stop'
|
|
for decl in iterator:
|
|
current_decl = decl
|
|
if isinstance(decl, pycparser.c_ast.Decl):
|
|
self._parse_decl(decl)
|
|
elif isinstance(decl, pycparser.c_ast.Typedef):
|
|
if not decl.name:
|
|
raise CDefError("typedef does not declare any name",
|
|
decl)
|
|
quals = 0
|
|
if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) and
|
|
decl.type.type.names[-1].startswith('__dotdotdot')):
|
|
realtype = self._get_unknown_type(decl)
|
|
elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
|
|
isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
|
|
isinstance(decl.type.type.type,
|
|
pycparser.c_ast.IdentifierType) and
|
|
decl.type.type.type.names[-1].startswith('__dotdotdot')):
|
|
realtype = self._get_unknown_ptr_type(decl)
|
|
else:
|
|
realtype, quals = self._get_type_and_quals(
|
|
decl.type, name=decl.name, partial_length_ok=True,
|
|
typedef_example="*(%s *)0" % (decl.name,))
|
|
self._declare('typedef ' + decl.name, realtype, quals=quals)
|
|
elif decl.__class__.__name__ == 'Pragma':
|
|
# skip pragma, only in pycparser 2.15
|
|
import warnings
|
|
warnings.warn(
|
|
"#pragma in cdef() are entirely ignored. "
|
|
"They should be removed for now, otherwise your "
|
|
"code might behave differently in a future version "
|
|
"of CFFI if #pragma support gets added. Note that "
|
|
"'#pragma pack' needs to be replaced with the "
|
|
"'packed' keyword argument to cdef().")
|
|
else:
|
|
raise CDefError("unexpected <%s>: this construct is valid "
|
|
"C but not valid in cdef()" %
|
|
decl.__class__.__name__, decl)
|
|
except CDefError as e:
|
|
if len(e.args) == 1:
|
|
e.args = e.args + (current_decl,)
|
|
raise
|
|
except FFIError as e:
|
|
msg = self._convert_pycparser_error(e, csource)
|
|
if msg:
|
|
e.args = (e.args[0] + "\n *** Err: %s" % msg,)
|
|
raise
|
|
|
|
def _add_constants(self, key, val):
|
|
if key in self._int_constants:
|
|
if self._int_constants[key] == val:
|
|
return # ignore identical double declarations
|
|
raise FFIError(
|
|
"multiple declarations of constant: %s" % (key,))
|
|
self._int_constants[key] = val
|
|
|
|
def _add_integer_constant(self, name, int_str):
|
|
int_str = int_str.lower().rstrip("ul")
|
|
neg = int_str.startswith('-')
|
|
if neg:
|
|
int_str = int_str[1:]
|
|
# "010" is not valid oct in py3
|
|
if (int_str.startswith("0") and int_str != '0'
|
|
and not int_str.startswith("0x")):
|
|
int_str = "0o" + int_str[1:]
|
|
pyvalue = int(int_str, 0)
|
|
if neg:
|
|
pyvalue = -pyvalue
|
|
self._add_constants(name, pyvalue)
|
|
self._declare('macro ' + name, pyvalue)
|
|
|
|
def _process_macros(self, macros):
|
|
for key, value in macros.items():
|
|
value = value.strip()
|
|
if _r_int_literal.match(value):
|
|
self._add_integer_constant(key, value)
|
|
elif value == '...':
|
|
self._declare('macro ' + key, value)
|
|
else:
|
|
raise CDefError(
|
|
'only supports one of the following syntax:\n'
|
|
' #define %s ... (literally dot-dot-dot)\n'
|
|
' #define %s NUMBER (with NUMBER an integer'
|
|
' constant, decimal/hex/octal)\n'
|
|
'got:\n'
|
|
' #define %s %s'
|
|
% (key, key, key, value))
|
|
|
|
def _declare_function(self, tp, quals, decl):
|
|
tp = self._get_type_pointer(tp, quals)
|
|
if self._options.get('dllexport'):
|
|
tag = 'dllexport_python '
|
|
elif self._inside_extern_python == '__cffi_extern_python_start':
|
|
tag = 'extern_python '
|
|
elif self._inside_extern_python == '__cffi_extern_python_plus_c_start':
|
|
tag = 'extern_python_plus_c '
|
|
else:
|
|
tag = 'function '
|
|
self._declare(tag + decl.name, tp)
|
|
|
|
def _parse_decl(self, decl):
|
|
node = decl.type
|
|
if isinstance(node, pycparser.c_ast.FuncDecl):
|
|
tp, quals = self._get_type_and_quals(node, name=decl.name)
|
|
assert isinstance(tp, model.RawFunctionType)
|
|
self._declare_function(tp, quals, decl)
|
|
else:
|
|
if isinstance(node, pycparser.c_ast.Struct):
|
|
self._get_struct_union_enum_type('struct', node)
|
|
elif isinstance(node, pycparser.c_ast.Union):
|
|
self._get_struct_union_enum_type('union', node)
|
|
elif isinstance(node, pycparser.c_ast.Enum):
|
|
self._get_struct_union_enum_type('enum', node)
|
|
elif not decl.name:
|
|
raise CDefError("construct does not declare any variable",
|
|
decl)
|
|
#
|
|
if decl.name:
|
|
tp, quals = self._get_type_and_quals(node,
|
|
partial_length_ok=True)
|
|
if tp.is_raw_function:
|
|
self._declare_function(tp, quals, decl)
|
|
elif (tp.is_integer_type() and
|
|
hasattr(decl, 'init') and
|
|
hasattr(decl.init, 'value') and
|
|
_r_int_literal.match(decl.init.value)):
|
|
self._add_integer_constant(decl.name, decl.init.value)
|
|
elif (tp.is_integer_type() and
|
|
isinstance(decl.init, pycparser.c_ast.UnaryOp) and
|
|
decl.init.op == '-' and
|
|
hasattr(decl.init.expr, 'value') and
|
|
_r_int_literal.match(decl.init.expr.value)):
|
|
self._add_integer_constant(decl.name,
|
|
'-' + decl.init.expr.value)
|
|
elif (tp is model.void_type and
|
|
decl.name.startswith('__cffi_extern_python_')):
|
|
# hack: `extern "Python"` in the C source is replaced
|
|
# with "void __cffi_extern_python_start;" and
|
|
# "void __cffi_extern_python_stop;"
|
|
self._inside_extern_python = decl.name
|
|
else:
|
|
if self._inside_extern_python !='__cffi_extern_python_stop':
|
|
raise CDefError(
|
|
"cannot declare constants or "
|
|
"variables with 'extern \"Python\"'")
|
|
if (quals & model.Q_CONST) and not tp.is_array_type:
|
|
self._declare('constant ' + decl.name, tp, quals=quals)
|
|
else:
|
|
_warn_for_non_extern_non_static_global_variable(decl)
|
|
self._declare('variable ' + decl.name, tp, quals=quals)
|
|
|
|
def parse_type(self, cdecl):
|
|
return self.parse_type_and_quals(cdecl)[0]
|
|
|
|
def parse_type_and_quals(self, cdecl):
|
|
ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
|
|
assert not macros
|
|
exprnode = ast.ext[-1].type.args.params[0]
|
|
if isinstance(exprnode, pycparser.c_ast.ID):
|
|
raise CDefError("unknown identifier '%s'" % (exprnode.name,))
|
|
return self._get_type_and_quals(exprnode.type)
|
|
|
|
def _declare(self, name, obj, included=False, quals=0):
|
|
if name in self._declarations:
|
|
prevobj, prevquals = self._declarations[name]
|
|
if prevobj is obj and prevquals == quals:
|
|
return
|
|
if not self._options.get('override'):
|
|
raise FFIError(
|
|
"multiple declarations of %s (for interactive usage, "
|
|
"try cdef(xx, override=True))" % (name,))
|
|
assert '__dotdotdot__' not in name.split()
|
|
self._declarations[name] = (obj, quals)
|
|
if included:
|
|
self._included_declarations.add(obj)
|
|
|
|
def _extract_quals(self, type):
|
|
quals = 0
|
|
if isinstance(type, (pycparser.c_ast.TypeDecl,
|
|
pycparser.c_ast.PtrDecl)):
|
|
if 'const' in type.quals:
|
|
quals |= model.Q_CONST
|
|
if 'volatile' in type.quals:
|
|
quals |= model.Q_VOLATILE
|
|
if 'restrict' in type.quals:
|
|
quals |= model.Q_RESTRICT
|
|
return quals
|
|
|
|
def _get_type_pointer(self, type, quals, declname=None):
|
|
if isinstance(type, model.RawFunctionType):
|
|
return type.as_function_pointer()
|
|
if (isinstance(type, model.StructOrUnionOrEnum) and
|
|
type.name.startswith('$') and type.name[1:].isdigit() and
|
|
type.forcename is None and declname is not None):
|
|
return model.NamedPointerType(type, declname, quals)
|
|
return model.PointerType(type, quals)
|
|
|
|
def _get_type_and_quals(self, typenode, name=None, partial_length_ok=False,
|
|
typedef_example=None):
|
|
# first, dereference typedefs, if we have it already parsed, we're good
|
|
if (isinstance(typenode, pycparser.c_ast.TypeDecl) and
|
|
isinstance(typenode.type, pycparser.c_ast.IdentifierType) and
|
|
len(typenode.type.names) == 1 and
|
|
('typedef ' + typenode.type.names[0]) in self._declarations):
|
|
tp, quals = self._declarations['typedef ' + typenode.type.names[0]]
|
|
quals |= self._extract_quals(typenode)
|
|
return tp, quals
|
|
#
|
|
if isinstance(typenode, pycparser.c_ast.ArrayDecl):
|
|
# array type
|
|
if typenode.dim is None:
|
|
length = None
|
|
else:
|
|
length = self._parse_constant(
|
|
typenode.dim, partial_length_ok=partial_length_ok)
|
|
# a hack: in 'typedef int foo_t[...][...];', don't use '...' as
|
|
# the length but use directly the C expression that would be
|
|
# generated by recompiler.py. This lets the typedef be used in
|
|
# many more places within recompiler.py
|
|
if typedef_example is not None:
|
|
if length == '...':
|
|
length = '_cffi_array_len(%s)' % (typedef_example,)
|
|
typedef_example = "*" + typedef_example
|
|
#
|
|
tp, quals = self._get_type_and_quals(typenode.type,
|
|
partial_length_ok=partial_length_ok,
|
|
typedef_example=typedef_example)
|
|
return model.ArrayType(tp, length), quals
|
|
#
|
|
if isinstance(typenode, pycparser.c_ast.PtrDecl):
|
|
# pointer type
|
|
itemtype, itemquals = self._get_type_and_quals(typenode.type)
|
|
tp = self._get_type_pointer(itemtype, itemquals, declname=name)
|
|
quals = self._extract_quals(typenode)
|
|
return tp, quals
|
|
#
|
|
if isinstance(typenode, pycparser.c_ast.TypeDecl):
|
|
quals = self._extract_quals(typenode)
|
|
type = typenode.type
|
|
if isinstance(type, pycparser.c_ast.IdentifierType):
|
|
# assume a primitive type. get it from .names, but reduce
|
|
# synonyms to a single chosen combination
|
|
names = list(type.names)
|
|
if names != ['signed', 'char']: # keep this unmodified
|
|
prefixes = {}
|
|
while names:
|
|
name = names[0]
|
|
if name in ('short', 'long', 'signed', 'unsigned'):
|
|
prefixes[name] = prefixes.get(name, 0) + 1
|
|
del names[0]
|
|
else:
|
|
break
|
|
# ignore the 'signed' prefix below, and reorder the others
|
|
newnames = []
|
|
for prefix in ('unsigned', 'short', 'long'):
|
|
for i in range(prefixes.get(prefix, 0)):
|
|
newnames.append(prefix)
|
|
if not names:
|
|
names = ['int'] # implicitly
|
|
if names == ['int']: # but kill it if 'short' or 'long'
|
|
if 'short' in prefixes or 'long' in prefixes:
|
|
names = []
|
|
names = newnames + names
|
|
ident = ' '.join(names)
|
|
if ident == 'void':
|
|
return model.void_type, quals
|
|
if ident == '__dotdotdot__':
|
|
raise FFIError(':%d: bad usage of "..."' %
|
|
typenode.coord.line)
|
|
tp0, quals0 = resolve_common_type(self, ident)
|
|
return tp0, (quals | quals0)
|
|
#
|
|
if isinstance(type, pycparser.c_ast.Struct):
|
|
# 'struct foobar'
|
|
tp = self._get_struct_union_enum_type('struct', type, name)
|
|
return tp, quals
|
|
#
|
|
if isinstance(type, pycparser.c_ast.Union):
|
|
# 'union foobar'
|
|
tp = self._get_struct_union_enum_type('union', type, name)
|
|
return tp, quals
|
|
#
|
|
if isinstance(type, pycparser.c_ast.Enum):
|
|
# 'enum foobar'
|
|
tp = self._get_struct_union_enum_type('enum', type, name)
|
|
return tp, quals
|
|
#
|
|
if isinstance(typenode, pycparser.c_ast.FuncDecl):
|
|
# a function type
|
|
return self._parse_function_type(typenode, name), 0
|
|
#
|
|
# nested anonymous structs or unions end up here
|
|
if isinstance(typenode, pycparser.c_ast.Struct):
|
|
return self._get_struct_union_enum_type('struct', typenode, name,
|
|
nested=True), 0
|
|
if isinstance(typenode, pycparser.c_ast.Union):
|
|
return self._get_struct_union_enum_type('union', typenode, name,
|
|
nested=True), 0
|
|
#
|
|
raise FFIError(":%d: bad or unsupported type declaration" %
|
|
typenode.coord.line)
|
|
|
|
def _parse_function_type(self, typenode, funcname=None):
|
|
params = list(getattr(typenode.args, 'params', []))
|
|
for i, arg in enumerate(params):
|
|
if not hasattr(arg, 'type'):
|
|
raise CDefError("%s arg %d: unknown type '%s'"
|
|
" (if you meant to use the old C syntax of giving"
|
|
" untyped arguments, it is not supported)"
|
|
% (funcname or 'in expression', i + 1,
|
|
getattr(arg, 'name', '?')))
|
|
ellipsis = (
|
|
len(params) > 0 and
|
|
isinstance(params[-1].type, pycparser.c_ast.TypeDecl) and
|
|
isinstance(params[-1].type.type,
|
|
pycparser.c_ast.IdentifierType) and
|
|
params[-1].type.type.names == ['__dotdotdot__'])
|
|
if ellipsis:
|
|
params.pop()
|
|
if not params:
|
|
raise CDefError(
|
|
"%s: a function with only '(...)' as argument"
|
|
" is not correct C" % (funcname or 'in expression'))
|
|
args = [self._as_func_arg(*self._get_type_and_quals(argdeclnode.type))
|
|
for argdeclnode in params]
|
|
if not ellipsis and args == [model.void_type]:
|
|
args = []
|
|
result, quals = self._get_type_and_quals(typenode.type)
|
|
# the 'quals' on the result type are ignored. HACK: we absure them
|
|
# to detect __stdcall functions: we textually replace "__stdcall"
|
|
# with "volatile volatile const" above.
|
|
abi = None
|
|
if hasattr(typenode.type, 'quals'): # else, probable syntax error anyway
|
|
if typenode.type.quals[-3:] == ['volatile', 'volatile', 'const']:
|
|
abi = '__stdcall'
|
|
return model.RawFunctionType(tuple(args), result, ellipsis, abi)
|
|
|
|
def _as_func_arg(self, type, quals):
|
|
if isinstance(type, model.ArrayType):
|
|
return model.PointerType(type.item, quals)
|
|
elif isinstance(type, model.RawFunctionType):
|
|
return type.as_function_pointer()
|
|
else:
|
|
return type
|
|
|
|
def _get_struct_union_enum_type(self, kind, type, name=None, nested=False):
|
|
# First, a level of caching on the exact 'type' node of the AST.
|
|
# This is obscure, but needed because pycparser "unrolls" declarations
|
|
# such as "typedef struct { } foo_t, *foo_p" and we end up with
|
|
# an AST that is not a tree, but a DAG, with the "type" node of the
|
|
# two branches foo_t and foo_p of the trees being the same node.
|
|
# It's a bit silly but detecting "DAG-ness" in the AST tree seems
|
|
# to be the only way to distinguish this case from two independent
|
|
# structs. See test_struct_with_two_usages.
|
|
try:
|
|
return self._structnode2type[type]
|
|
except KeyError:
|
|
pass
|
|
#
|
|
# Note that this must handle parsing "struct foo" any number of
|
|
# times and always return the same StructType object. Additionally,
|
|
# one of these times (not necessarily the first), the fields of
|
|
# the struct can be specified with "struct foo { ...fields... }".
|
|
# If no name is given, then we have to create a new anonymous struct
|
|
# with no caching; in this case, the fields are either specified
|
|
# right now or never.
|
|
#
|
|
force_name = name
|
|
name = type.name
|
|
#
|
|
# get the type or create it if needed
|
|
if name is None:
|
|
# 'force_name' is used to guess a more readable name for
|
|
# anonymous structs, for the common case "typedef struct { } foo".
|
|
if force_name is not None:
|
|
explicit_name = '$%s' % force_name
|
|
else:
|
|
self._anonymous_counter += 1
|
|
explicit_name = '$%d' % self._anonymous_counter
|
|
tp = None
|
|
else:
|
|
explicit_name = name
|
|
key = '%s %s' % (kind, name)
|
|
tp, _ = self._declarations.get(key, (None, None))
|
|
#
|
|
if tp is None:
|
|
if kind == 'struct':
|
|
tp = model.StructType(explicit_name, None, None, None)
|
|
elif kind == 'union':
|
|
tp = model.UnionType(explicit_name, None, None, None)
|
|
elif kind == 'enum':
|
|
if explicit_name == '__dotdotdot__':
|
|
raise CDefError("Enums cannot be declared with ...")
|
|
tp = self._build_enum_type(explicit_name, type.values)
|
|
else:
|
|
raise AssertionError("kind = %r" % (kind,))
|
|
if name is not None:
|
|
self._declare(key, tp)
|
|
else:
|
|
if kind == 'enum' and type.values is not None:
|
|
raise NotImplementedError(
|
|
"enum %s: the '{}' declaration should appear on the first "
|
|
"time the enum is mentioned, not later" % explicit_name)
|
|
if not tp.forcename:
|
|
tp.force_the_name(force_name)
|
|
if tp.forcename and '$' in tp.name:
|
|
self._declare('anonymous %s' % tp.forcename, tp)
|
|
#
|
|
self._structnode2type[type] = tp
|
|
#
|
|
# enums: done here
|
|
if kind == 'enum':
|
|
return tp
|
|
#
|
|
# is there a 'type.decls'? If yes, then this is the place in the
|
|
# C sources that declare the fields. If no, then just return the
|
|
# existing type, possibly still incomplete.
|
|
if type.decls is None:
|
|
return tp
|
|
#
|
|
if tp.fldnames is not None:
|
|
raise CDefError("duplicate declaration of struct %s" % name)
|
|
fldnames = []
|
|
fldtypes = []
|
|
fldbitsize = []
|
|
fldquals = []
|
|
for decl in type.decls:
|
|
if (isinstance(decl.type, pycparser.c_ast.IdentifierType) and
|
|
''.join(decl.type.names) == '__dotdotdot__'):
|
|
# XXX pycparser is inconsistent: 'names' should be a list
|
|
# of strings, but is sometimes just one string. Use
|
|
# str.join() as a way to cope with both.
|
|
self._make_partial(tp, nested)
|
|
continue
|
|
if decl.bitsize is None:
|
|
bitsize = -1
|
|
else:
|
|
bitsize = self._parse_constant(decl.bitsize)
|
|
self._partial_length = False
|
|
type, fqual = self._get_type_and_quals(decl.type,
|
|
partial_length_ok=True)
|
|
if self._partial_length:
|
|
self._make_partial(tp, nested)
|
|
if isinstance(type, model.StructType) and type.partial:
|
|
self._make_partial(tp, nested)
|
|
fldnames.append(decl.name or '')
|
|
fldtypes.append(type)
|
|
fldbitsize.append(bitsize)
|
|
fldquals.append(fqual)
|
|
tp.fldnames = tuple(fldnames)
|
|
tp.fldtypes = tuple(fldtypes)
|
|
tp.fldbitsize = tuple(fldbitsize)
|
|
tp.fldquals = tuple(fldquals)
|
|
if fldbitsize != [-1] * len(fldbitsize):
|
|
if isinstance(tp, model.StructType) and tp.partial:
|
|
raise NotImplementedError("%s: using both bitfields and '...;'"
|
|
% (tp,))
|
|
tp.packed = self._options.get('packed')
|
|
if tp.completed: # must be re-completed: it is not opaque any more
|
|
tp.completed = 0
|
|
self._recomplete.append(tp)
|
|
return tp
|
|
|
|
def _make_partial(self, tp, nested):
|
|
if not isinstance(tp, model.StructOrUnion):
|
|
raise CDefError("%s cannot be partial" % (tp,))
|
|
if not tp.has_c_name() and not nested:
|
|
raise NotImplementedError("%s is partial but has no C name" %(tp,))
|
|
tp.partial = True
|
|
|
|
def _parse_constant(self, exprnode, partial_length_ok=False):
|
|
# for now, limited to expressions that are an immediate number
|
|
# or positive/negative number
|
|
if isinstance(exprnode, pycparser.c_ast.Constant):
|
|
s = exprnode.value
|
|
if '0' <= s[0] <= '9':
|
|
s = s.rstrip('uUlL')
|
|
try:
|
|
if s.startswith('0'):
|
|
return int(s, 8)
|
|
else:
|
|
return int(s, 10)
|
|
except ValueError:
|
|
if len(s) > 1:
|
|
if s.lower()[0:2] == '0x':
|
|
return int(s, 16)
|
|
elif s.lower()[0:2] == '0b':
|
|
return int(s, 2)
|
|
raise CDefError("invalid constant %r" % (s,))
|
|
elif s[0] == "'" and s[-1] == "'" and (
|
|
len(s) == 3 or (len(s) == 4 and s[1] == "\\")):
|
|
return ord(s[-2])
|
|
else:
|
|
raise CDefError("invalid constant %r" % (s,))
|
|
#
|
|
if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
|
|
exprnode.op == '+'):
|
|
return self._parse_constant(exprnode.expr)
|
|
#
|
|
if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
|
|
exprnode.op == '-'):
|
|
return -self._parse_constant(exprnode.expr)
|
|
# load previously defined int constant
|
|
if (isinstance(exprnode, pycparser.c_ast.ID) and
|
|
exprnode.name in self._int_constants):
|
|
return self._int_constants[exprnode.name]
|
|
#
|
|
if (isinstance(exprnode, pycparser.c_ast.ID) and
|
|
exprnode.name == '__dotdotdotarray__'):
|
|
if partial_length_ok:
|
|
self._partial_length = True
|
|
return '...'
|
|
raise FFIError(":%d: unsupported '[...]' here, cannot derive "
|
|
"the actual array length in this context"
|
|
% exprnode.coord.line)
|
|
#
|
|
if isinstance(exprnode, pycparser.c_ast.BinaryOp):
|
|
left = self._parse_constant(exprnode.left)
|
|
right = self._parse_constant(exprnode.right)
|
|
if exprnode.op == '+':
|
|
return left + right
|
|
elif exprnode.op == '-':
|
|
return left - right
|
|
elif exprnode.op == '*':
|
|
return left * right
|
|
elif exprnode.op == '/':
|
|
return self._c_div(left, right)
|
|
elif exprnode.op == '%':
|
|
return left - self._c_div(left, right) * right
|
|
elif exprnode.op == '<<':
|
|
return left << right
|
|
elif exprnode.op == '>>':
|
|
return left >> right
|
|
elif exprnode.op == '&':
|
|
return left & right
|
|
elif exprnode.op == '|':
|
|
return left | right
|
|
elif exprnode.op == '^':
|
|
return left ^ right
|
|
#
|
|
raise FFIError(":%d: unsupported expression: expected a "
|
|
"simple numeric constant" % exprnode.coord.line)
|
|
|
|
def _c_div(self, a, b):
|
|
result = a // b
|
|
if ((a < 0) ^ (b < 0)) and (a % b) != 0:
|
|
result += 1
|
|
return result
|
|
|
|
def _build_enum_type(self, explicit_name, decls):
|
|
if decls is not None:
|
|
partial = False
|
|
enumerators = []
|
|
enumvalues = []
|
|
nextenumvalue = 0
|
|
for enum in decls.enumerators:
|
|
if _r_enum_dotdotdot.match(enum.name):
|
|
partial = True
|
|
continue
|
|
if enum.value is not None:
|
|
nextenumvalue = self._parse_constant(enum.value)
|
|
enumerators.append(enum.name)
|
|
enumvalues.append(nextenumvalue)
|
|
self._add_constants(enum.name, nextenumvalue)
|
|
nextenumvalue += 1
|
|
enumerators = tuple(enumerators)
|
|
enumvalues = tuple(enumvalues)
|
|
tp = model.EnumType(explicit_name, enumerators, enumvalues)
|
|
tp.partial = partial
|
|
else: # opaque enum
|
|
tp = model.EnumType(explicit_name, (), ())
|
|
return tp
|
|
|
|
def include(self, other):
|
|
for name, (tp, quals) in other._declarations.items():
|
|
if name.startswith('anonymous $enum_$'):
|
|
continue # fix for test_anonymous_enum_include
|
|
kind = name.split(' ', 1)[0]
|
|
if kind in ('struct', 'union', 'enum', 'anonymous', 'typedef'):
|
|
self._declare(name, tp, included=True, quals=quals)
|
|
for k, v in other._int_constants.items():
|
|
self._add_constants(k, v)
|
|
|
|
def _get_unknown_type(self, decl):
|
|
typenames = decl.type.type.names
|
|
if typenames == ['__dotdotdot__']:
|
|
return model.unknown_type(decl.name)
|
|
|
|
if typenames == ['__dotdotdotint__']:
|
|
if self._uses_new_feature is None:
|
|
self._uses_new_feature = "'typedef int... %s'" % decl.name
|
|
return model.UnknownIntegerType(decl.name)
|
|
|
|
if typenames == ['__dotdotdotfloat__']:
|
|
# note: not for 'long double' so far
|
|
if self._uses_new_feature is None:
|
|
self._uses_new_feature = "'typedef float... %s'" % decl.name
|
|
return model.UnknownFloatType(decl.name)
|
|
|
|
raise FFIError(':%d: unsupported usage of "..." in typedef'
|
|
% decl.coord.line)
|
|
|
|
def _get_unknown_ptr_type(self, decl):
|
|
if decl.type.type.type.names == ['__dotdotdot__']:
|
|
return model.unknown_ptr_type(decl.name)
|
|
raise FFIError(':%d: unsupported usage of "..." in typedef'
|
|
% decl.coord.line)
|