...
pytd.pandas_td.create_engine
(url, con=None, header=True, show_progress=5.0, clear_progress=True)[source]
...
| Code Block |
|---|
>>> import pytd.pandas_td as td
>>> con = td.connect(apikey=apikey, endpoint="https://api.treasuredata.com")
>>> engine = td.create_engine("presto:sample_datasets") |
pytd.pandas_td.read_td_query
pytd.pandas_td.read_td_query(query, engine, index_col=None, parse_dates=None, distributed_join=False, params=None)[source]
...
Return type
pandas.DataFrame
pytd.pandas_td.read_td_job
pytd.pandas_td.read_td_job(job_id, engine, index_col=None, parse_dates=None)[source]
...
Return type
pandas.DataFrame
pytd.pandas_td.read_td_table
pytd.pandas_td.read_td_table(table_name, engine, index_col=None, parse_dates=None, columns=None, time_range=None, limit=10000)[source]
...
Return type
pandas.DataFrame
pytd.pandas_td.read_td
pytd.pandas_td.read_td(query, engine, index_col=None, parse_dates=None, distributed_join=False, params=None)[source]
...
Return type
pandas.DataFrame
pytd.pandas_td.to_td
pytd.pandas_td.to_td(frame, name, con, if_exists='fail', time_col=None, time_index=None, index=True, index_label=None, chunksize=10000, date_format=None, writer='bulk_import', **kwargs)[source]
...
|
|---|
pytd.pandas_td.ipyhton.MagicContext
classpytd.pandas_td.ipython.MagicContext[source]
__init__()[source]
Initialize self. See help(type(self)) for accurate signature.
connect()[source]
pytd.pandas_td.ipyhton.MagicTable
classpytd.pandas_td.ipython.MagicTable(table)[source]
__init__(table)[source]
Initialize self. See help(type(self)) for accurate signature.
pytd.pandas_td.ipyhton.get_td_magic_context()
pytd.pandas_td.ipyhton.TDMagics
classpytd.pandas_td.ipython.TDMagics(shell)[source]
__init__(shell)[source]
Create a configurable given a config config.
Parameters
config (Config) – If this is empty, default values are used. If config is a
Configinstance, it will be used to configure the instance.parent (Configurable instance, optional) – The parent Configurable instance of this object.
Notes
Subclasses of Configurable must call the __init__() method of Configurable before doing anything else and using super():
| Code Block |
|---|
class MyConfigurable(Configurable):
def __init__(self, config=None):
super(MyConfigurable, self).__init__(config=config)
# Then any other code you need to finish initialization.
|
This ensures that instances will be configured properly.
pytd.pandas_td.ipython.DatabasesMagics
classpytd.pandas_td.ipython.DatabasesMagics(shell)[source]
td_databases(pattern)
List databases in the form of pandas.DataFrame.
| Code Block |
|---|
%td_databases [<database_name_pattern>]
|
Parameters
<database_name_pattern> (string, optional) – List databases matched to a given pattern. If not given, all existing databases will be listed.
Returns
Return type
pandas.DataFrame
Examples
| Code Block |
|---|
In [1]: %load_ext pytd.pandas_td.ipython
In [2]: %td_databases sample
Out[2]:
name count permission created_at updated_at
0 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 348124 administrator 2019-01-23 05:48:11+00:00 2019-01-23 05:48:11+00:00
1 yyyyyyyyy 0 administrator 2017-12-14 07:52:34+00:00 2017-12-14 07:52:34+00:00
2 zzzzzzzzzzzzz 0 administrator 2016-05-25 23:12:06+00:00 2016-05-25 23:12:06+00:00
...
In [3]: %td_databases sample
Out[3]:
name count permission created_at updated_at
0 sampledb 2 administrator 2014-04-11 22:29:38+00:00 2014-04-11 22:29:38+00:00
1 sample_xxxxxxxx 2 administrator 2017-06-02 23:37:41+00:00 2017-06-02 23:37:41+00:00
2 sample_datasets 8812278 query_only 2014-10-04 01:13:11+00:00 2018-03-16 04:59:06+00:00
...
|
magics= {'cell': {}, 'line': {'td_databases': 'td_databases'}}
registered= True
pytd.pandas_td.ipython.TableMagics
classpytd.pandas_td.ipython.TablesMagics(shell)[source]
td_tables(pattern)
List tables in databases.
| Code Block |
|---|
%td_tables [<table_identifier_pattern>]
|
Parameters
<table_identifier_pattern> (string, optional) – List tables matched to a given pattern. Table identifier is represented as database_name.table_name. If not given, all existing tables will be listed.
Returns
Return type
pandas.DataFrame
Examples
| Code Block |
|---|
In [1]: %load_ext pytd.pandas_td.ipython
In [2]: %td_tables
Out[2]:
db_name name count estimated_storage_size last_log_timestamp created_at
0 xxxxx_demo_aa customer_test 70 1047 2018-02-05 06:20:32+00:00 2018-02-05 06:20:24+00:00
1 xxxxx_demo_aa email_log 0 0 1970-01-01 00:00:00+00:00 2018-02-05 07:19:57+00:00
2 yy_wf topk_similar_items 10598 134208 2018-04-16 09:23:57+00:00 2018-04-16 09:59:48+00:00
...
In [3]: %td_tables sample
Out[3]:
db_name name count estimated_storage_size last_log_timestamp created_at
0 xx_test aaaaaaaa_sample 0 0 1970-01-01 00:00:00+00:00 2015-10-20 17:37:40+00:00
1 sampledb sampletbl 2 843 1970-01-01 00:00:00+00:00 2014-04-11 22:30:08+00:00
2 zzzz_test_db sample_output_tab 4 889 2018-06-06 08:26:20+00:00 2018-06-06 08:27:12+00:00
...
|
magics= {'cell': {}, 'line': {'td_tables': 'td_tables'}}
registered= True
pytd.pandas_td.ipython.JobMagics
classpytd.pandas_td.ipython.JobsMagics(shell)[source]
td_jobs(line)
List job activities in an account.
| Code Block |
|---|
%td_jobs
|
Returns
Return type
pandas.DataFrame
Examples
| Code Block |
|---|
In [1]: %load_ext pytd.pandas_td.ipython
In [2]: %td_jobs
Out[2]:
status job_id type start_at query
0 error 448650806 hive 2019-04-12 05:33:36+00:00 with null_samples as (\n select\n id,\n ...
1 success 448646994 presto 2019-04-12 05:23:29+00:00 -- read_td_query\n-- set session distributed_j...
2 success 448646986 presto 2019-04-12 05:23:27+00:00 -- read_td_query\n-- set session distributed_j...
...
|
magics= {'cell': {}, 'line': {'td_jobs': 'td_jobs'}}
registered= True
pytd.pandas_td.ipython.UseMagics
classpytd.pandas_td.ipython.UseMagics(shell)[source]
td_use(line)
Use a specific database.
This magic pushes all table names in a specified database into the current namespace.
| Code Block |
|---|
%td_use [<database_name>] |
Parameters
<database_name> (string) – Database name.
Examples
| Code Block |
|---|
In [1]: %load_ext pytd.pandas_td.ipython
In [2]: %td_use sample_datasets
INFO: import nasdaq
INFO: import www_access
In [3]: nasdaq # describe table columns in the form of DataFrame
Out[3]: <pytd.pandas_td.ipython.MagicTable at 0x117651908>
|
magics= {'cell': {}, 'line': {'td_use': 'td_use'}}
registered= True