Python Yield

Python yield
Mail & Popo: yangjunwei@

1

Agenda
• why thinking in yield?

• yield

• yield

• yield

2

why thinking in yield?

3


db.begin()
if check_error1_happen(db):
db.rollback()
return Response(“error1 happens”)

db.rollback()

db.table.update()
....
db.commit()

return Response(“success”)

4


• pythonic

• bug

5

@contextmanager
def transaction(conn):
class Job(object): trans = Job()
def __init__(self): conn.begin()
self._finished = False
try:
yield trans
def is_finished(self): except:
return self._finished conn.rollback()
raise

def finish(self): if trans.is_finished():
self._finished = True conn.commit()
else:
conn.rollback()

6


with transaction(db) as trans:


db.table.update()
....
trans.ﬁnish()

return Response(“success”)

7


main main
A yield function

B C

yield function

8

yield

Using a yield expression in a function definition is sufficient to
cause that definition to create a generator function instead of
a normal function.

10

yield

Using a yield expression in a function definition is sufficient to
cause that definition to create a generator function instead of
a normal function.

>>> def f():
... yield "hello world"
...
>>> type(f())
<type 'generator'>

10

yield

That generator then controls the execution of a generator
function.

11

yield

That generator then controls the execution of a generator
function.

• send()

• next() ==> send(None)

• throw()

• close() ==> throw(GeneratorExit)

11

yield

yield is a expression, not a statement. New in python 2.5

12

yield

yield is a expression, not a statement. New in python 2.5

>>> def f():
... a = yield "hello"
... print a
...
>>> b = f()
>>> b.send(None)
'hello'
>>> b.send("world")
world
StopIteration

12

yield

generator is also a iterator.

13

yield

generator is also a iterator.

>>> a = f()
>>> a is a.__iter__()
True

13

yield

generator object

14

yield

generator object

>>> def f():
... yield "hello world"
...
>>> a = f()
>>> a.next()
“hello world”
>>> a.next()
StopIteration
>>> a.next()
StopIteration

14

yield
tstate_head

next next
PyThreadState PyThreadState ...
frame

PyFrameObject

f_back
PyFrameObject

f_back
python
...

16

yield

code block
compile
PyCodeObject

MAKE_FUNCTION
PyFunctionObject
CALL_FUNCTION

PyFrameObject

17

yield
[frameobject.c]

typedef struct {
...
struct _frame *f_back;
PyCodeObject *f_code;
PyObject *f_builtins;
PyObject *f_globals;
PyObject *f_locals;
PyObject *f_valuestack;
PyObject *f_stacktop;
...
int f_lasti;
...
} PyFrameObject;

18

yield

[codeobject.c]

typedef struct {
...
int co_ﬂags;
...
PyObject *co_code;
PyObject *co_consts;
PyObject *co_names;
...
} PyCodeObject;

19

yield

[a.py] [b.py]

def f(): def f():
return “hello world” yield “hello world”

f() f()

PyCodeObject <==> code block

20

yield

4 0 LOAD_CONST 0 (<code object f ..>)
3 MAKE_FUNCTION 0
6 STORE_NAME 0 (f)

7 9 LOAD_NAME 0 (f)
12 CALL_FUNCTION 0
15 POP_TOP
16 LOAD_CONST 1 (None)
19 RETURN_VALUE

dis.dis(main_code)

21

yield

5 0 LOAD_CONST 1 ('hello world')
3 RETURN_VALUE

5 0 LOAD_CONST 1 ('hello world')
3 YIELD_VALUE
4 POP_TOP
5 LOAD_CONST 0 (None)
8 RETURN_VALUE

dis.dis(f_code)

22

yield

return generator

23

yield

[ceval.c] CALL_FUNCTION

f = PyFrame_New(tstate, co, globals, locals);
...
if (co->co_ﬂags & CO_GENERATOR) {
...
f->f_back = NULL;
...
return PyGen_New(f);
}

retval = PyEval_EvalFrameEx(f,0);

24

yield

[frameobject.c]

PyFrame_New(PyThreadState *tstate,
PyCodeObject *code, PyObject *globals, PyObject *locals){
...
PyFrameObject *back = tstate->frame;
PyFrameObject *f;
f = PyObject_GC_Resize(PyFrameObject, f, extras);
...
f->f_code = code;
...
f->f_back = back;
f->f_tstate = tstate;
return f;
}

25

yield

[ceval.c]

PyObject *PyEval_EvalFrameEx(PyFrameObject *f,
int throwﬂag){
...
PyThreadState *tstate = PyThreadState_GET();
tstate->frame = f;
...
// exec opcode
...
tstate->frame = f->f_back;
return retval;
}

26

yield
tstate_head

next
PyThreadState ...
frame
A frame
f_back
B frame

f_back
...

27

yield
tstate_head

next
PyThreadState ...
frame
A frame new frame
f_back
B frame

f_back
...

27

yield
tstate_head

next
PyThreadState ...
frame
A frame new frame
f_back
f_back
B frame

f_back
...

27

yield
tstate_head

next
PyThreadState ...
frame
A frame
f_back
B frame

f_back
...
yield

28

yield
tstate_head

next
PyThreadState ...
frame
A frame new frame
f_back
B frame

f_back
...
yield

28

yield
tstate_head

next
PyThreadState ...
frame
A frame new frame
f_back
f_back
B frame

f_back
...
yield

28

yield
[genobject.c]

static PyObject * gen_send_ex(PyGenObject *gen, PyObject *arg, int
exc) {
PyFrameObject *f = gen->gi_frame;
f->f_back = tstate->frame;
...

if (f->f_lasti == -1) {
....
} else {
result = arg ? arg : Py_None;

Py_INCREF(result);

*(f->f_stacktop++) = result;
}
...
result = PyEval_EvalFrameEx(f, exc);
}

29

yield
tstate_head

next
PyThreadState ...
frame
A frame(caller) new frame
f_back
B frame(creator)

f_back
...
send

30

yield
tstate_head

next
PyThreadState ...
frame
A frame(caller) new frame
f_back
f_back
B frame(creator)

f_back
...
send

30

yield

resume frame

31

yield
[ceval.c]

PyObject *PyEval_EvalFrameEx(PyFrameObject *f,
int throwﬂag){
...

next_instr = ﬁrst_instr + f->f_lasti + 1;

stack_pointer = f->f_stacktop;

assert(stack_pointer != NULL);

f->f_stacktop = NULL;
...
case YIELD_VALUE:
retval = POP();
f->f_stacktop = stack_pointer;
why = WHY_YIELD;
goto fast_yield;
}

32

yield

[genobject.c]

static PyObject *gen_throw(PyGenObject *gen,
PyObject *args){
...

PyErr_Restore(typ, val, tb);

return gen_send_ex(gen, Py_None, 1);
}

throw

33

yield

def close(self):
try:
self.throw(GeneratorExit)
except (GeneratorExit, StopIteration):
pass
else:
raise RuntimeError("generator ignored GeneratorExit")
# Other exceptions are not caught

close

34

• xrange

• with_statement“ ”

• python Trampoline

• generator coroutine

36

xrange

[rangeobject.c]

static PyObject *
rangeiter_next(rangeiterobject *r)
{

if (r->index < r->len)

return PyInt_FromLong(r->start + (r->index++) * r-
>step);

return NULL;
}

xrange

38

xrange

def xrange(start, stop, step=1):
i = start - step
while True:
i = i + step

if i < stop:
yield i
else:
return

yield

39

xrange

l = [x for x in xrange(10)]
g = (x * x for x in xrange(10))

list generator

40

xrange

l = [x for x in xrange(10)]
g = (x * x for x in xrange(10))

list generator

def f():
for x in xrange(10):
yield x * x

40

with_statement“ ”

41

“with_statement”

with with

42


with with

with EXPR as VAR:
BLOCK

42


mgr = (EXPR)
exit = type(mgr).__exit__ # Not calling it yet
value = type(mgr).__enter__(mgr)
exc = True
try:
try:
VAR = value # Only if "as VAR" is present
BLOCK
except:
exc = False
if not exit(mgr, *sys.exc_info()):
raise
ﬁnally:
if exc:
exit(mgr, None, None, None)

43

class Transaction(object):
def __init__(self, db):
self._db = db
self._trans = Job()

def __enter__(self):
self._db.begin()
return self._trans

def __exit__(self, type, value, traceback):
if type is not None:
self._db.rollback()
else:
self._db.commit()

yield transaction

44

@contextmanager
def transaction(conn):
trans = Job()
conn.begin()

try:
yield trans
except:
conn.rollback()
raise

if trans.is_ﬁnished():
conn.commit()
else:
conn.rollback()

45

python Trampoline

46

python Trampoline

def f(n):
if n < 2:
return n
else:
return n + f(n - 1)

>>> f(999)
499500
>>> f(1000)
RuntimeError: maximum recursion depth exceeded

47

python Trampoline

sys.getrecursionlimit()

Return the current value of the recursion limit, the
maximum depth of the Python interpreter stack. This
limit prevents inﬁnite recursion from causing an
overﬂow of the C stack and crashing Python. It can be
set by setrecursionlimit().

48

python Trampoline

49

python Trampoline

f(n) f(n)
control
f(n-1) f(n-1)

f(n-2) scheduler f(n-2)
f_back
... ...

f(1) f(1)

50

python Trampoline

• let generator "call" other generator by yielding
the generator they wish to invoke.

• Any non-generator value yielded by a
generator is returned to the generator that
"called" the one yielding the value.

51

python Trampoline

def f(n):
if n < 2:
yield n
else:
yield n + (yield f(n-1))

52

python Trampoline

def trampoline(main):
stack = []
value = main
while True:
if type(value) is types.GeneratorType:
stack.append(value)
value = stack[-1].next()
else:
stack.pop()
if stack:
value = stack[-1].send(value)
else:
return value

53

generator coroutine

54

generator coroutine

Coroutine( )
" " " "

55

generator coroutine

Coroutine( )
" " " "

Python's generator functions are almost coroutines.

55

generator coroutine

generator coroutine

56

generator coroutine

generator coroutine

• coroutine “call” coroutine

• coroutine scheduler

56

generator coroutine

coroutine-based Python networking library

• Eventlet(http://eventlet.net/)

• Gevent(http://www.gevent.org/)

57

Resources & References

• python references: yield expression

• python

• PEP 342: Coroutines via Enhanced Generators

• PEP 343: the with statement

58

Python Yield

More Related Content

What's hot

Viewers also liked

Similar to Python Yield

Recently uploaded

Python Yield

Editor's Notes