
    ZThn                        d dl Z d dlmZ d dlmZmZ d dlmZ d dlmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d d	lmZ d dlZd d
lmZ d dlmZ d dlmZmZ d dlmZ d dl m!Z!m"Z"m#Z#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z* d dl+mZ d dl,m-Z-  eej\                  j^                  j`                        ZdefdZ1defdZ2defdZ3defdZ4defdZ5defdZ6defdZ7defdZ8defdZ9defdZ:defdZ;y)    N)contextmanager)datetimetimezone)sleep)CallableList)MockFixture)!IngestWithConfigDataStreamRequest)
FlowConfig)IngestionConfig)MockChannel)
_mock_path)ProtobufMaxSizeExceededErrorSiftAPIDeprecationWarning)IngestionValidationError)create_flow_configscreate_ingestion_config"get_ingestion_config_by_client_keyget_ingestion_config_flows)ChannelConfigChannelDataTypedouble_value)TelemetryConfig)IngestionServicemockerc                 h   | j                   j                  t        d      d_        t	        dt        dt        j                        g      }t        dd|g	      }t        d
dd      }| j                  t        t                    }||_        | j                  t        t                    }|j                  t              g|_        t        t               |      }t         fd       } |       5  |j#                         5 }|j$                  dk(  sJ t'        d      D ]W  }	|j)                  dt+        j,                  t.        j0                        dt3        t5        j4                               dgd       Y j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       j6                  dk(  sJ 	 ddd        |       5  |j#                         5 }|j$                  dk(  sJ t'        d      D ]W  }	|j)                  dt+        j,                  t.        j0                        dt3        t5        j4                               dgd       Y j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       j6                  dk(  sJ 	 ddd        |       5  |j#                  d      5 }|j$                  dk(  sJ t'        d      D ]W  }	|j)                  dt+        j,                  t.        j0                        dt3        t5        j4                               dgd       Y j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       j6                  dk(  sJ 	 ddd        |       5  |j#                  d      5 }t'        d      D ]T  }	|j=                  dt+        j,                  t.        j0                        t3        t5        j4                               gd       V j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       j6                  dk(  sJ 	 ddd        |       5  |j#                         5 }t'        d      D ]T  }	|j=                  dt+        j,                  t.        j0                        t3        t5        j4                               gd       V j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       j6                  dk(  sJ 	 ddd        |       5  |j#                         5 }t'        d      D ]T  }	|j=                  dt+        j,                  t.        j0                        t3        t5        j4                               gd       V j6                  dk(  sJ t9        |j:                        d k(  sJ t?        j@                  tB              5   # 1 sw Y   3xY w# 1 sw Y   &xY w# 1 sw Y   pxY w# 1 sw Y   cxY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   MxY w# 1 sw Y   @xY w# 1 sw Y   nxY w	 ddd       n# 1 sw Y   nxY wt9        j:                        dk(  sJ j6                  dk(  sJ 	 ddd       n# 1 sw Y   nxY w |       5  | jE                         d!tF        d"tH        tJ           d#tL        ffd$}
t?        j@                  tB              5  |j#                  |
%      5 }t'        d      D ]T  }	|j=                  dt+        j,                  t.        j0                        t3        t5        j4                               gd       V  # 1 sw Y   nxY w	 ddd       n# 1 sw Y   nxY wjO                          t9        j:                        d k(  sJ j6                  dk(  sJ 	 ddd       n# 1 sw Y   nxY w |       5  | jE                         d!tF        d"tH        tJ           d#tL        ffd&}
t?        j@                  tB              5  |j#                  |
%      5 }t'        d      D ]T  }	|j=                  dt+        j,                  t.        j0                        t3        t5        j4                               gd       V  # 1 sw Y   nxY w	 ddd       n# 1 sw Y   nxY wjO                          t9        j:                        dk(  sJ j6                  dk(  sJ 	 ddd       y# 1 sw Y   yxY w)'z
    Ensures that the ingestion method is being called the expected amount of times
    when using the buffered method of ingestion.
    ingestNreadings
my-channelname	data_typer!   channelsmy-assetingestion-client-key
asset_nameingestion_client_keyflowsingestion-config-idasset-id
client-keyingestion_config_idasset_id
client_keyc               3   6   K   d   j                          y wN
reset_mockmock_ingests   d/home/www/backend.miabetepe.com/venv/lib/python3.12/site-packages/sift_py/ingestion/_service_test.pymock_ctx_managerzCtest_ingestion_service_buffered_ingestion.<locals>.mock_ctx_managerI              i'  channel_namevalue	flow_name	timestampchannel_values
   r   i)       iP     i      i     ip  i  iX  errrequests_flushc                              y r3    )rJ   rK   rL   on_error_spys      r8   on_errorz;test_ingestion_service_buffered_ingestion.<locals>.on_error   s     N    )rP   c                 "              |        y r3   rN   )rJ   rK   rL   on_error_flush_spys      r8   rP   z;test_ingestion_service_buffered_ingestion.<locals>.on_error   s      HrQ   )(patchobjectr   return_valuer   r   r   DOUBLEr   IngestionConfigPbr   r   r   as_pbFlowConfigPbr   r   buffered_ingestion_buffer_sizerangetry_ingest_flowsr   nowr   utcr   random
call_countlen_bufferingest_flowspytestraises	ExceptionstubBaseExceptionr   r
   r   assert_called_once)r   readings_flowtelemetry_configmock_ingestion_config'mock_get_ingestion_config_by_client_keymock_get_ingestion_config_flowsingestion_servicer9   r[   _rP   r7   rS   rO   s              @@@r8   )test_ingestion_service_buffered_ingestionrs       s    ,,%%&6AK#K!)00
M '3o .1 /5ll56/+ <Q+8&,ll:>X3Y&Z#4A4G4G4U3V#0(8HI! ! 
	 ,113 	87I%22e;;;6] 	"33%/%-\\(,,%?-9LQWQ^Q^Q`Dab+	 ))R///)112a777	8" %%+++%,( 
	 ,113 	:7I%22e;;;6] 	"33%/%-\\(,,%?-9LQWQ^Q^Q`Dab+	 ))R///)112c999	:$ %%+++',* 
	 ,11#6 	::L%22c9995\ 	"33%/%-\\(,,%?-9LQWQ^Q^Q`Dab+	 ))R///)112c999	:" %%+++%,( 
	 +11#6 	::L5\ "//%/%-\\(,,%?+7+H*I ))Q...)112c999	: %%***+  
	 +113 	87I5\ "//%/%-\\(,,%?+7+H*I ))Q...)112a777	8 %%***+  
	 +113 	7I5\ "//%/%-\\(,,%?+7+H*I ))Q...)112c999y) W	8 	8, ,*	: 	:, ,,	: 	:, ,*	: 	:+ +"	8 	8+ +<  	 	 	  %--.!333%%***%+ + +( 
	 +{{}		*./P*Q	[c	 ]]9% 
	"55x5H 	L^u A&33)3)1hll)C/;FMMO/L.M 	 	 	
	 
	 
	 	'')%--.#555%%***/+ + +2 
	 +#[[]		*./P*Q	[c	 ]]9% 
	"55x5H 	L^u A&33)3)1hll)C/;FMMO/L.M 	 	 	
	 
	 
	 	'')%--.!333%%***1+ + +sb  6W(B"W*W(X$B"W5X0XB"X%XX6 BX)/X6Y)BY8Y!Z42B'Y3YW%	 W((W25W?	:XXX	XX&)X3	.X66Y Y	YYY&"Y3*	Z43Y<	82Z44Z=A	_.^#A$^^^	_.^&	"A_.._7A	d(
cA$cc
c	d(c 	Ad((d1c                 4   t        ddd      }t        j                  t        d      5  t	        ddt
        j                  	      }d
d
d
       t        dg      }t        d|j                  |g      }| j                  t        t                    }d
|_        | j                  t        t                    }||_        | j                  t        t                    }|j!                  t"              g|_        t%               }t'        ||      }	|j)                  ||j*                  |j,                  |j.                  d
       |	j0                  |j2                     j4                  d   j2                  |j2                  k(  sJ |	j0                  |j2                     j4                  d   j6                  |j6                  k(  sJ |	j0                  |j2                     j4                  d   j8                  |j8                  k(  sJ t
        j:                  |_        | j                  t        t<                    }
d
|
_        |j?                          ||_        t        j@                  tB              5  t'        ||      }	d
d
d
       t
        j                  |_        |
j?                          t'        ||      }	|
jE                          |	j0                  |j2                     j4                  d   j2                  |j2                  k(  sJ |	j0                  |j2                     j4                  d   j6                  |j6                  k(  sJ |	j0                  |j2                     j4                  d   j8                  |j8                  k(  sJ y
# 1 sw Y   ;xY w# 1 sw Y   xY w)aV  
    Tests modifying existing channel configs in telemetry config. If a channel config
    is modified in a telemetry config after it has already been used for ingestion
    then we should create a new flow. If a user modifies a channel back to a previous
    version (same component and name), then we should re-use an existing channel.
    my-ingestion-config-idmy-ingestion-configmy-asset-idr/   r1   r0   	componentmatch	channel_aAr!   ry   r"   Nflow_ar#   my-asset-namer'   channelconfigr   )#rX   rf   warnsr   r   r   rW   r   r   r/   rT   r   r   rV   r   r   rY   rZ   r   r   assert_called_once_withr(   r*   r)   flow_configs_by_namer!   r$   ry   r"   STRINGr   r5   rg   r   assert_not_called)r   rn   r|   r   rm   ro   mock_create_ingestion_configrp   mock_channelrq   mock_create_flow_configss              r8   6test_ingestion_service_modify_existing_channel_configsr      sS    .4( 
/{	C 
!%,,
	
 F
 '"2FFh /5ll56/+ <@+8#)<<
;R0S#T 0E -&,ll:>X3Y&Z#4:LL4N3O#0=L(
 !88##-- 11&++>GGJOOS\SaSaaaa..v{{;DDQGQQ		 	..v{{;DDQGQQ		
 *00I%||J7J,KL,0)+668;P+8 
/	0 
, #

 *00I'') ) ..011&++>GGJOOS\SaSaaaa..v{{;DDQGQQ		 	..v{{;DDQGQQ		{
 
D
 
s   N *N N
Nc                    t        ddd      }t        j                  t        d      5  t	        ddt
        j                  	      }d d d        t        d
g      }t        d|j                  |g      }| j                  t        t                    }d |_        | j                  t        t                    }||_        | j                  t        t                    }|j!                  t"              g|_        t%               }t'        ||      }	t        dt	        dt
        j                        g      }
| j                  t        t(                    }d |_        |	j*                  j-                  d      J |	j/                  |
       |j1                  ||j                  |
f       |	j*                  d   |
k(  sJ t        dt	        dt
        j                        g      }t        j2                  t4              5  |	j/                  |       d d d        |	j7                  |       |	j*                  d   |k(  sJ |	j*                  d   |
k7  sJ y # 1 sw Y   	xY w# 1 sw Y   PxY w)Nru   rv   rw   rx   ry   rz   r|   r}   r~   r   r#   r   r'   r   my_new_flownew_channelfoobar)rX   rf   r   r   r   r   rW   r   r   r/   rT   r   r   rV   r   r   rY   rZ   r   r   r   r   gettry_create_flowr   rg   r   create_flow)r   rn   r|   r   rm   ro   r   rp   r   rq   new_flow_configr   new_flow_config_name_collisions                r8   (test_ingestion_service_register_new_flowr   c  sR   -4( 
/{	C 
!%,,
	
 F
 '"2FFh /5ll56/+ <@+8#)<<
;R0S#T 0E -&,ll:>X3Y&Z#4:LL4N3O#0=L(
 !mM?CYCY&Z%[O  &||J7J,KL,0)1155mDLLL%%o644+??/AS 11-@OSSS &0mHo>T>T&U%V&" 
/	0 J))*HIJ !!"@A11-@Dbbbb11-@OSSS
 
rJ Js   I:III$c                   
 | j                   j                  t        d      
d
_        t	        dt        dt        j                        g      }t        dd|g	      }t        d
dd      }| j                  t        t                    }||_        | j                  t        t                    }|j                  t              g|_        t        t               |      }t         
fd       } |       5  |j#                  d      5 }|j$                  dk(  sJ t'        d      D ]W  }	|j)                  dt+        j,                  t.        j0                        dt3        t5        j4                               dgd       Y 
j6                  dk(  sJ t9        |j:                        dk(  sJ t=        d       
j6                  dk(  sJ t9        |j:                        dk(  sJ 	 ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)z
    Test for timeout based flush mechanism in buffered ingestion. If buffer hasn't been flushed
    after a certain time then the buffer will be automatically flushed.
    r   Nr   r   r    r#   r%   r&   r'   r+   r,   r-   r.   c               3   6   K   d   j                          y wr3   r4   r6   s   r8   r9   zQtest_ingestion_service_buffered_ingestion_flush_timeout.<locals>.mock_ctx_manager  r:   r;      )flush_interval_secr<   i  r=   r@      rE      r   )rT   rU   r   rV   r   r   r   rW   r   rX   r   r   r   rY   rZ   r   r   r[   r\   r]   r^   r   r_   r   r`   r   ra   rb   rc   rd   r   )r   rl   rm   rn   ro   rp   rq   r9   r[   rr   r7   s             @r8   7test_ingestion_service_buffered_ingestion_flush_timeoutr     s    ,,%%&6AK#K!)00
M '3o .1 /5ll56/+ <Q+8&,ll:>X3Y&Z#4A4G4G4U3V#0(8HI! ! 
	 811Q1G 	8K]%22e;;;5\ 	"33%/%-\\(,,%?-9LQWQ^Q^Q`Dab+	 ))Q...)112c999 !H))Q...)112a777'	88 8	8 	88 8s%   4G=CG1 G=1G:	6G==Hc                 >   t        ddd      }t        dt        j                        }t	        d|g      }t        d	|g
      }| j                  t        t                    }d |_	        | j                  t        t                    }||_	        t               }t        ||      }|j                  ||j                  |j                  |j!                         d        t#        j$                  t&        d      5  |j)                          d d d        y # 1 sw Y   y xY w)Nru   rv   rw   rx   r|   r    r   r#   r   r(   r*   r   zcan not be updated at runtimerz   )rX   r   r   rW   r   r   rT   r   r   rV   r   r   r   r   r(   r*   hashrf   rg   r   r   )	r   rn   r|   r   rm   ro   r   r   rq   s	            r8   Dtest_ingestion_service_telemetry_config_without_ingestion_client_keyr     s   -4( !((I
 F
 '"h
 /5ll56/+ <@+8#)<<
;R0S#T 0E -=L(
 !88## 
/7V	W (%%'( ( (s   9DDc                 ~   t        dt        dt        j                        g      }t	        dd|g      }t        dd	d
      }| j                  t        t                    }||_	        t               }t        ||d      }|j                  du sJ |j                  d|ik(  sJ |j                  t               k(  sJ y)z
    Tests that use_lazy_flow_creation flag is set to True when force_lazy_flow_creation=True.
    And that flow_configs_by_name and flow_configs_created are set correctly
    r   r|   r    r#   r%   my-client-keyr'   r+   r,   r.   Tr   r   force_lazy_flow_creationN)r   r   r   rW   r   rX   rT   r   r   rV   r   r   use_lazy_flow_creationr   flow_configs_createdset)r   r   rm   rn   ro   r   svcs          r8   >test_ingestion_service_lazy_flow_creation_flag_set_when_forcedr   !  s    
  )00
F ',h .1" /5ll56/+ <Q+8=L
!%C %%---##&'9999##su,,,rQ   c                    t        dt        dt        j                        g      }t	        dd|g      }t        dd	d
      }| j                  j                  t        d      }t        d      |g|_
        t               }t        ||      }|j                  du sJ |j                  d|ik(  sJ |j                  t               k(  sJ y)z
    Tests that use_lazy_flow_creation flag is set to True when ProtobufMaxSizeExceededError is raised.
    And that flow_configs_by_name and flow_configs_created are set correctly
    r   r|   r    r#   r%   r   r'   r+   r,   r.   _get_or_create_ingestion_configzMessage too larger   TN)r   r   r   rW   r   rX   rT   rU   r   r   side_effectr   r   r   r   r   )r   r   rm   rn   #mock_get_or_create_ingestion_configr   r   s          r8   Rtest_ingestion_service_lazy_flow_creation_flag_set_when_protobuf_max_size_exceededr   N  s      )00
F ',h .1" +1,,*=*=;+' 	%%897'3
 =L
C
 %%---##&'9999##su,,,rQ   c                    t        dt        dt        j                        g      }t	        dd|g      }t        dd	d
      }| j                  t        t                    }||_	        | j                  t        t                    }|j                  t              g|_	        t               }t        ||      }|j                  du sJ t!        |j"                  j%                               dgk(  sJ |j&                  dhk(  sJ y)z
    Tests that use_lazy_flow_creation flag is False by default.
    And that flow_configs_by_name and flow_configs_created are set correctly
    r   r|   r    r#   r%   r   r'   r+   r,   r.   r   FN)r   r   r   rW   r   rX   rT   r   r   rV   r   rY   rZ   r   r   r   listr   keysr   )r   r   rm   rn   ro   rp   r   r   s           r8   ?test_ingestion_service_lazy_flow_creation_flag_false_by_defaultr     s   
  )00
F ',h .1" /5ll56/+ <Q+8&,ll:>X3Y&Z#4:LL4N3O#0=L
C
 %%...((--/0XJ>>>##z111rQ   c                 &   t        dt        dt        j                        g      }t        dt        dt        j                        g      }t        d||g      }t        d	d
|j                               }| j                  t        t                    }d|_        | j                  t        t                    }||_        t               }t        ||      }|j                  du sJ |j                   ||dk(  sJ |j"                  ddhk(  sJ y)z
    Tests flow_configs_by_name and flow_configs_created population when using generated client key
    with lazy flow creation disabled. And that flow_configs_by_name and flow_configs_created are set correctly
    r   r|   r    r#   flow_b	channel_br%   r   r+   r,   r.   Nr   Fr   r   )r   r   r   rW   r   r   rX   r   rT   r   r   rV   r   r   r   r   r   r   	r   r   r   rm   rn   ro   r   r   r   s	            r8   ;test_ingestion_service_generated_client_key_with_lazy_flowsr     s,   
  )00
F  )00
F 'v
 .1#((* /5ll56/+ <@+8#)<<
;R0S#T 0E -=L
C
 %%...##&F'KKKK##(';;;;rQ   c                 4   t        dt        dt        j                        g      }t        dt        dt        j                        g      }t        d||g      }t        d	d
|j                               }| j                  t        t                    }d|_        | j                  t        t                    }||_        t               }t        ||d      }|j                  du sJ |j                   ||dk(  sJ |j"                  t%               k(  sJ y)z
    Tests flow_configs_by_name and flow_configs_created population when using generated client key
    with lazy flow creation enabled.
    r   r|   r    r#   r   r   r%   r   r+   r,   r.   NTr   r   )r   r   r   rW   r   r   rX   r   rT   r   r   rV   r   r   r   r   r   r   r   r   s	            r8   Ctest_ingestion_service_generated_client_key_with_lazy_flows_enabledr     s,   
  )00
F  )00
F 'v
 .1#((* /5ll56/+ <@+8#)<<
;R0S#T 0E -=L
!%C %%---##&F'KKKK##su,,,rQ   c                    t        dt        dt        j                        g      }t        dt        dt        j                        g      }t        dd||g	      }t        d
dd      }| j                  t        t                    }||_
        t               }t        ||d      }|j                  du sJ |j                  ||dk(  sJ |j                  t!               k(  sJ y)z
    Tests flow_configs_by_name and flow_configs_created population when using custom client key
    with lazy flow creation enabled.
    r   r|   r    r#   r   r   r%   z
custom-keyr'   r+   r,   r.   Tr   r   N)r   r   r   rW   r   r   rX   rT   r   r   rV   r   r   r   r   r   r   )r   r   r   rm   rn   ro   r   r   s           r8   @test_ingestion_service_custom_client_key_with_lazy_flows_enabledr     s   
  )00
F  )00
F ')v .1 /5ll56/+ <Q+8=L
!%C %%---##&F'KKKK##su,,,rQ   )<ra   
contextlibr   r   r   timer   typingr   r   rf   pytest_mockr	   sift.ingest.v1.ingest_pb2r
   /sift.ingestion_configs.v2.ingestion_configs_pb2r   rZ   r   rX   "sift_py.ingestion._internal.ingestsift_py#sift_py._internal.test_util.channelr   sift_py._internal.test_util.fnr   _mock_path_impsift_py.errorr   r   !sift_py.ingestion._internal.errorr   ,sift_py.ingestion._internal.ingestion_configr   r   r   r   sift_py.ingestion.channelr   r   r   "sift_py.ingestion.config.telemetryr   sift_py.ingestion.flowsift_py.ingestion.servicer   	ingestion	_internalr   rs   r   r   r   r   r   r   r   r   r   r   rN   rQ   r8   <module>r      s    % '  !  # G V ` ) ; G Q F  S R > - 6G--77>>?
P+k P+fm; m`FT[ FTRB8K B8J-(Q\ -(`*-; *-Z.-.-b,2K ,2^5< 5<p6-P[ 6-r4-[ 4-rQ   